7517
Cd
clanker-and-the-dev / files
clanker-and-the-dev / .agents / skills / add-new-comic / scripts / generate_panel.py
35.8 KB
1from __future__ import annotations 2 3import random 4import re 5from pathlib import Path 6 7from PIL import Image, ImageDraw, ImageOps 8 9from models import ComicSpec, GenerationConfig, PanelSpec 10 11_PIPELINE = None 12_PIPELINE_KEY = None 13 14STYLE_PROMPT = ( 15 "single plain wordless comic illustration, clean green line sketch on solid black background, " 16 "thin unfilled strokes, simple Dev, boxy Clanker, comic art only" 17) 18 19# Normalized crop boxes for art-reference/training-reference.png. The sheet is 20# intentionally clean and black-backed; cropping lets IP-Adapter see large 21# character examples instead of a tiny downscaled full character sheet. 22TRAINING_REFERENCE_CROPS: dict[str, tuple[float, float, float, float]] = { 23 "clanker_neutral_body": (0.013, 0.026, 0.101, 0.271), 24 "clanker_happy_body": (0.106, 0.026, 0.193, 0.271), 25 "clanker_side_body": (0.382, 0.026, 0.430, 0.270), 26 "clanker_at_laptop": (0.720, 0.043, 0.859, 0.272), 27 "clanker_neutral_face": (0.019, 0.297, 0.100, 0.492), 28 "clanker_happy_face": (0.119, 0.298, 0.210, 0.492), 29 "clanker_thinking_face": (0.322, 0.300, 0.403, 0.492), 30 "clanker_concerned_face": (0.423, 0.299, 0.503, 0.492), 31 "clanker_overheated_face": (0.632, 0.300, 0.727, 0.492), 32 "clanker_heart_face": (0.749, 0.299, 0.829, 0.492), 33 "dev_neutral_body": (0.024, 0.523, 0.080, 0.729), 34 "dev_mug_body": (0.105, 0.523, 0.161, 0.729), 35 "dev_laptop_body": (0.286, 0.536, 0.370, 0.727), 36 "dev_seated_laptop": (0.387, 0.537, 0.508, 0.728), 37 "dev_thumbs_body": (0.544, 0.523, 0.618, 0.726), 38 "dev_excited_body": (0.796, 0.525, 0.885, 0.723), 39 "dev_running_body": (0.896, 0.526, 0.989, 0.723), 40 "dev_neutral_face": (0.020, 0.775, 0.084, 0.915), 41 "dev_happy_face": (0.106, 0.775, 0.170, 0.915), 42 "dev_confused_face": (0.193, 0.776, 0.257, 0.915), 43 "dev_frown_face": (0.279, 0.776, 0.343, 0.914), 44 "dev_panic_face": (0.443, 0.768, 0.537, 0.914), 45 "dev_love_face": (0.752, 0.762, 0.906, 0.917), 46} 47 48# Normalized crop boxes for art-reference/ip-character-reference.png. The 49# source file is a good high-quality style anchor, but the full image contains 50# two large black character cards and baseline shadows that SDXL tends to copy 51# as stray background panels. Auto-cropping feeds just the characters to 52# IP-Adapter while keeping the high-resolution proportions/style signal. 53IP_CHARACTER_REFERENCE_CROPS: dict[str, tuple[float, float, float, float]] = { 54 "ip_dev_full": (0.080, 0.085, 0.270, 0.555), 55 "ip_clanker_full": (0.385, 0.088, 0.620, 0.560), 56} 57 58BASE_REFERENCE_CROPS = [ 59 "dev_neutral_body", 60 "clanker_neutral_body", 61 "dev_laptop_body", 62 "clanker_happy_body", 63] 64 65# Extra pose crops used only for auto ControlNet guides. These are cut from 66# the clean art-reference sheets so the guide itself has the same proportions 67# as the target comic instead of a generic stick-figure sketch. 68CONTROL_REFERENCE_SPRITES: dict[str, tuple[str, tuple[float, float, float, float]]] = { 69 **{name: ("training-reference.png", box) for name, box in TRAINING_REFERENCE_CROPS.items()}, 70 "dev_pose_overwhelmed": ("dev-poses-sheet.png", (0.845, 0.285, 0.940, 0.470)), 71 "dev_pose_collapsed": ("dev-poses-sheet.png", (0.345, 0.590, 0.485, 0.775)), 72 "dev_pose_dead": ("dev-poses-sheet.png", (0.505, 0.570, 0.640, 0.775)), 73 "dev_pose_slumped": ("dev-poses-sheet.png", (0.170, 0.600, 0.325, 0.755)), 74 "dev_pose_zombie": ("dev-poses-sheet.png", (0.655, 0.600, 0.790, 0.790)), 75} 76 77_CONTROL_SPRITE_CACHE: dict[tuple[str, str], Image.Image] = {} 78 79 80def _torch_dtype(torch, dtype_name: str): 81 return { 82 "float16": torch.float16, 83 "bfloat16": torch.bfloat16, 84 "float32": torch.float32, 85 }[dtype_name] 86 87 88def _pipeline_key(config: GenerationConfig) -> tuple[object, ...]: 89 return ( 90 config.modelId, 91 config.enableControlNet, 92 config.controlNetModelId, 93 config.ipAdapterRepo, 94 config.ipAdapterSubfolder, 95 config.ipAdapterWeight, 96 config.ipAdapterScale, 97 config.device, 98 config.torchDtype, 99 ) 100 101 102def load_pipeline(config: GenerationConfig): 103 global _PIPELINE, _PIPELINE_KEY 104 key = _pipeline_key(config) 105 if _PIPELINE is not None and _PIPELINE_KEY == key: 106 return _PIPELINE 107 108 import torch 109 from diffusers import ControlNetModel, StableDiffusionXLControlNetPipeline, StableDiffusionXLPipeline 110 111 if config.device == "cuda" and not torch.cuda.is_available(): 112 raise RuntimeError("CUDA was requested but torch.cuda.is_available() is false") 113 114 dtype = _torch_dtype(torch, config.torchDtype) 115 variant = "fp16" if config.torchDtype == "float16" else None 116 117 if config.enableControlNet: 118 controlnet = ControlNetModel.from_pretrained( 119 config.controlNetModelId, 120 torch_dtype=dtype, 121 use_safetensors=True, 122 ) 123 pipe = StableDiffusionXLControlNetPipeline.from_pretrained( 124 config.modelId, 125 controlnet=controlnet, 126 torch_dtype=dtype, 127 use_safetensors=True, 128 variant=variant, 129 ) 130 else: 131 pipe = StableDiffusionXLPipeline.from_pretrained( 132 config.modelId, 133 torch_dtype=dtype, 134 use_safetensors=True, 135 variant=variant, 136 ) 137 138 pipe.load_ip_adapter( 139 config.ipAdapterRepo, 140 subfolder=config.ipAdapterSubfolder, 141 weight_name=config.ipAdapterWeight, 142 ) 143 pipe.set_ip_adapter_scale(config.ipAdapterScale) 144 145 if config.device == "cuda" and config.enableSequentialCpuOffload and hasattr(pipe, "enable_sequential_cpu_offload"): 146 # SDXL + IP-Adapter + ControlNet does not fit reliably in 12 GB with 147 # model-level offload. Sequential offload is slower but keeps the full 148 # quality path usable on the target RTX 3080 Ti. 149 pipe.enable_sequential_cpu_offload() 150 elif config.device == "cuda" and config.enableModelCpuOffload: 151 pipe.enable_model_cpu_offload() 152 else: 153 pipe.to(config.device) 154 155 if config.enableVaeSlicing and hasattr(pipe, "enable_vae_slicing"): 156 pipe.enable_vae_slicing() 157 if config.enableVaeTiling and hasattr(pipe, "enable_vae_tiling"): 158 pipe.enable_vae_tiling() 159 if config.enableAttentionSlicing and hasattr(pipe, "enable_attention_slicing"): 160 pipe.enable_attention_slicing() 161 162 _PIPELINE = pipe 163 _PIPELINE_KEY = key 164 return pipe 165 166 167def release_pipeline(device: str = "cuda") -> None: 168 global _PIPELINE, _PIPELINE_KEY 169 if _PIPELINE is None: 170 return 171 172 pipe = _PIPELINE 173 _PIPELINE = None 174 _PIPELINE_KEY = None 175 176 try: 177 if hasattr(pipe, "maybe_free_model_hooks"): 178 pipe.maybe_free_model_hooks() 179 except Exception: 180 pass 181 try: 182 pipe.to("cpu") 183 except Exception: 184 pass 185 186 del pipe 187 188 import gc 189 190 gc.collect() 191 if device == "cuda": 192 try: 193 import torch 194 195 torch.cuda.empty_cache() 196 torch.cuda.ipc_collect() 197 except Exception: 198 pass 199 200 201def compact_visual_text(text: str) -> str: 202 replacements = { 203 "Dev, a human software developer with simple rectangular glasses": "Dev", 204 "a human software developer with simple rectangular glasses": "Dev", 205 "Clanker, a helpful boxy robot assistant with screen eyes and antenna": "Clanker", 206 "Clanker, a boxy robot assistant with screen eyes and antenna": "Clanker", 207 "a helpful boxy robot assistant with screen eyes and antenna": "Clanker", 208 "a boxy robot assistant with screen eyes and antenna": "Clanker", 209 "pure black terminal room": "plain black room", 210 "pure black terminal workspace": "plain black workspace", 211 "dark neon terminal room": "plain black room", 212 "black neon terminal room": "plain black room", 213 "black terminal room": "plain black room", 214 "dark neon terminal workspace": "plain black workspace", 215 "black neon terminal workspace": "plain black workspace", 216 "black terminal workspace": "plain black workspace", 217 "green screen glow": "terminal lines", 218 "small green suggestion glow": "small cursor", 219 "boot-up glow": "boot-up cursor", 220 "terminal boot glow": "boot cursor", 221 "glowing laptop": "laptop", 222 "glowing": "terminal", 223 "glows": "shows", 224 "glow": "terminal line", 225 "in the same neon terminal room with a confident robot grin on its screen eyes while": "while", 226 "in a plain black room while": "while", 227 "in a black terminal room while": "while", 228 "sparks and tiny green debris implying the source code vanished": "sparks and debris", 229 "sparse dark upper area reserved for speech bubbles": "empty dark top", 230 "dark upper area reserved for a speech bubble": "empty dark top", 231 "empty dark upper area": "empty top", 232 "empty top area": "empty top", 233 "terminal line energy": "simple motion lines", 234 "terminal cursor": "small cursor", 235 "terminal laptop": "laptop", 236 "Dev on the left": "Dev left", 237 "Clanker on the right": "Clanker right", 238 "on the left at the laptop": "left at laptop", 239 "on the right in a triumphant pose": "right triumphant", 240 "on the left": "left", 241 "on the right": "right", 242 "overwhelmed Dev on the left": "Dev left panicked", 243 "laptop screen centered": "laptop center", 244 "leans in helpfully with one claw raised": "stands helpfully", 245 "laptop foreground": "laptop", 246 "foreground": "", 247 } 248 for old, new in replacements.items(): 249 text = text.replace(old, new) 250 text = re.sub(r"\b(Dev|Clanker),\s*", r"\1 ", text) 251 return re.sub(r"\s+", " ", text).strip() 252 253 254def limit_words(text: str, max_words: int) -> str: 255 words = text.split() 256 return " ".join(words[:max_words]) 257 258 259def build_prompt(comic: ComicSpec, panel: PanelSpec, panel_number: int) -> str: 260 # CLIP token limits are tight. Put the house style first, then a compact 261 # scene description; the reference-derived ControlNet guide carries layout. 262 parts = [ 263 STYLE_PROMPT, 264 f"{panel.shot} shot, full bodies", 265 limit_words(compact_visual_text(panel.artPrompt), 22), 266 ] 267 return ". ".join(part.strip().rstrip(".") for part in parts if part and part.strip()) + "." 268 269 270def _panel_text(panel: PanelSpec) -> str: 271 dialogue = " ".join(line.text for line in panel.dialogue) 272 return " ".join( 273 part 274 for part in [panel.artPrompt, panel.composition or "", panel.caption or "", dialogue] 275 if part 276 ).lower() 277 278 279def _has_any(text: str, words: tuple[str, ...]) -> bool: 280 return any(word in text for word in words) 281 282 283def reference_crop_names_for_panel(panel: PanelSpec) -> list[str]: 284 text = _panel_text(panel) 285 names = list(BASE_REFERENCE_CROPS) 286 287 if _has_any(text, ("laptop", "typing", "terminal", "desk", "seated", "nvim")): 288 names.extend(["dev_seated_laptop", "clanker_at_laptop"]) 289 if _has_any(text, ("excited", "wow", "hype", "hyped", "celebrat", "triumphant", "proud", "confident")): 290 names.extend(["dev_excited_body", "dev_happy_face", "clanker_happy_face"]) 291 if _has_any(text, ("panic", "panicked", "overwhelmed", "chaos", "sweat", "holy", "sparks", "overclock")): 292 names.extend(["dev_panic_face", "clanker_overheated_face"]) 293 if _has_any(text, ("dead", "collapsed", "floor", "burnout", "flat", "concerned")): 294 names.extend(["dev_frown_face", "clanker_concerned_face"]) 295 if _has_any(text, ("thinking", "confused", "curious", "question")): 296 names.extend(["dev_confused_face", "clanker_thinking_face"]) 297 if _has_any(text, ("coffee", "mug")): 298 names.append("dev_mug_body") 299 if _has_any(text, ("running", "run")): 300 names.append("dev_running_body") 301 if _has_any(text, ("love", "heart")): 302 names.extend(["dev_love_face", "clanker_heart_face"]) 303 304 # Keep order stable but remove duplicates. 305 return list(dict.fromkeys(names)) 306 307 308def square_on_black(image: Image.Image, size: int = 512) -> Image.Image: 309 image = ImageOps.exif_transpose(image).convert("RGB") 310 image = ImageOps.contain(image, (size, size), method=Image.Resampling.LANCZOS) 311 canvas = Image.new("RGB", (size, size), "#020806") 312 canvas.paste(image, ((size - image.width) // 2, (size - image.height) // 2)) 313 return canvas 314 315 316def crop_normalized(image: Image.Image, box: tuple[float, float, float, float]) -> Image.Image: 317 width, height = image.size 318 left, top, right, bottom = box 319 crop_box = ( 320 max(0, round(left * width)), 321 max(0, round(top * height)), 322 min(width, round(right * width)), 323 min(height, round(bottom * height)), 324 ) 325 return image.crop(crop_box) 326 327 328def _expanded_bbox(box: tuple[int, int, int, int], size: tuple[int, int], padding: int = 10) -> tuple[int, int, int, int]: 329 left, top, right, bottom = box 330 width, height = size 331 return ( 332 max(0, left - padding), 333 max(0, top - padding), 334 min(width, right + padding), 335 min(height, bottom + padding), 336 ) 337 338 339def control_reference_sprite(project_root: Path, name: str) -> Image.Image | None: 340 spec = CONTROL_REFERENCE_SPRITES.get(name) 341 if spec is None: 342 return None 343 344 cache_key = (str(project_root.resolve()), name) 345 cached = _CONTROL_SPRITE_CACHE.get(cache_key) 346 if cached is not None: 347 return cached.copy() 348 349 source_name, box = spec 350 path = project_root / "art-reference" / source_name 351 if not path.exists(): 352 return None 353 354 crop = crop_normalized(Image.open(path).convert("RGB"), box) 355 alpha = ImageOps.grayscale(crop).point(lambda p: 0 if p < 28 else min(255, p * 2)) 356 bbox = alpha.getbbox() 357 if bbox is None: 358 return None 359 360 bbox = _expanded_bbox(bbox, crop.size, padding=12) 361 alpha = alpha.crop(bbox) 362 sprite = Image.new("RGBA", alpha.size, (255, 255, 255, 0)) 363 sprite.putalpha(alpha) 364 _CONTROL_SPRITE_CACHE[cache_key] = sprite 365 return sprite.copy() 366 367 368def paste_control_sprite( 369 image: Image.Image, 370 sprite: Image.Image, 371 center_x: int, 372 baseline_y: int, 373 target_height: int, 374) -> None: 375 if target_height <= 0: 376 return 377 scale = target_height / sprite.height 378 target_width = max(1, round(sprite.width * scale)) 379 sprite = sprite.resize((target_width, target_height), Image.Resampling.LANCZOS) 380 x = round(center_x - target_width / 2) 381 y = round(baseline_y - target_height) 382 image.paste(sprite, (x, y), sprite) 383 384 385def training_reference_crops(image: Image.Image, panel: PanelSpec) -> list[Image.Image]: 386 crops: list[Image.Image] = [] 387 for name in reference_crop_names_for_panel(panel): 388 box = TRAINING_REFERENCE_CROPS.get(name) 389 if box is not None: 390 crops.append(square_on_black(crop_normalized(image, box))) 391 return crops 392 393 394def ip_character_reference_crops(image: Image.Image) -> list[Image.Image]: 395 return [square_on_black(crop_normalized(image, box)) for box in IP_CHARACTER_REFERENCE_CROPS.values()] 396 397 398def load_reference_images(config: GenerationConfig, panel: PanelSpec, project_root: Path) -> list[Image.Image]: 399 images: list[Image.Image] = [] 400 for raw_path in config.referenceImages: 401 path = (project_root / raw_path).resolve() 402 if not path.exists(): 403 raise FileNotFoundError(f"Reference image not found: {path}") 404 image = Image.open(path).convert("RGB") 405 if config.referenceCropMode == "auto" and path.name == "training-reference.png": 406 images.extend(training_reference_crops(image, panel)) 407 elif config.referenceCropMode == "auto" and path.name == "ip-character-reference.png": 408 images.extend(ip_character_reference_crops(image)) 409 else: 410 images.append(square_on_black(image)) 411 412 if not images: 413 raise ValueError("At least one reference image is required for IP-Adapter conditioning") 414 return images 415 416 417def make_reference_contact_sheet(images: list[Image.Image], size: int = 384) -> Image.Image: 418 tiles = [square_on_black(image, size=size) for image in images] 419 cols = 1 if len(tiles) == 1 else 2 420 rows = (len(tiles) + cols - 1) // cols 421 sheet = Image.new("RGB", (cols * size, rows * size), "#020806") 422 for index, image in enumerate(tiles): 423 x = (index % cols) * size 424 y = (index // cols) * size 425 sheet.paste(image, (x, y)) 426 return sheet 427 428 429def make_ip_adapter_image_embeds(pipe, reference_images: list[Image.Image], torch, device, do_classifier_free_guidance: bool): 430 projection = getattr(getattr(pipe.unet, "encoder_hid_proj", None), "image_projection_layers", None) 431 if not projection or len(projection) != 1: 432 return None 433 434 from diffusers.models.embeddings import ImageProjection 435 436 output_hidden_state = not isinstance(projection[0], ImageProjection) 437 positive_embeds = [] 438 negative_embeds = [] 439 with torch.no_grad(): 440 for image in reference_images: 441 positive, negative = pipe.encode_image(image, device, 1, output_hidden_state) 442 positive_embeds.append(positive.detach().cpu()) 443 negative_embeds.append(negative.detach().cpu()) 444 if torch.cuda.is_available(): 445 torch.cuda.empty_cache() 446 447 positive = torch.stack(positive_embeds, dim=0).mean(dim=0) 448 negative = torch.stack(negative_embeds, dim=0).mean(dim=0) 449 450 # Keep the CLIP image encoder from occupying VRAM during ControlNet/UNet 451 # denoising. Diffusers' model CPU offload does not always evict it after a 452 # manual encode_image call. 453 if getattr(pipe, "image_encoder", None) is not None: 454 try: 455 pipe.image_encoder.to("cpu") 456 except Exception: 457 pass 458 if torch.cuda.is_available(): 459 torch.cuda.empty_cache() 460 461 # The original IP-Adapter projection expects [batch, image_count, embed_dim]. 462 # Hidden-state variants already include the token dimension. 463 if not output_hidden_state: 464 positive = positive.unsqueeze(1) 465 negative = negative.unsqueeze(1) 466 467 if do_classifier_free_guidance: 468 return [torch.cat([negative, positive], dim=0)] 469 return [positive] 470 471 472def _clause_position(text: str, name: str, default: float) -> float: 473 clauses = re.split(r"[.;,]", text) 474 for clause in clauses: 475 if name not in clause: 476 continue 477 if "left" in clause: 478 return 0.30 479 if "right" in clause: 480 return 0.70 481 if "center" in clause or "middle" in clause: 482 return 0.50 483 if "foreground" in clause: 484 return 0.36 if name == "dev" else 0.64 485 return default 486 487 488def draw_laptop(draw: ImageDraw.ImageDraw, width: int, height: int, x: float, y: float, scale: float, stroke: int): 489 screen_w = int(170 * scale) 490 screen_h = int(100 * scale) 491 keyboard_h = int(34 * scale) 492 x0 = int(x - screen_w / 2) 493 y0 = int(y - screen_h) 494 x1 = int(x + screen_w / 2) 495 y1 = int(y) 496 draw.rectangle((x0, y0, x1, y1), outline="white", width=stroke) 497 draw.line((x0 + stroke, y1, x1 - stroke, y1), fill="white", width=stroke) 498 draw.polygon( 499 [ 500 (x0 - int(24 * scale), y1 + keyboard_h), 501 (x1 + int(24 * scale), y1 + keyboard_h), 502 (x1, y1), 503 (x0, y1), 504 ], 505 outline="white", 506 ) 507 draw.line((x0 + int(46 * scale), y0 + int(45 * scale), x1 - int(46 * scale), y0 + int(45 * scale)), fill="white", width=stroke) 508 draw.line((x0 + int(70 * scale), y0 + int(28 * scale), x0 + int(45 * scale), y0 + int(50 * scale)), fill="white", width=stroke) 509 draw.line((x1 - int(70 * scale), y0 + int(28 * scale), x1 - int(45 * scale), y0 + int(50 * scale)), fill="white", width=stroke) 510 511 512def draw_token_meter(draw: ImageDraw.ImageDraw, width: int, height: int, stroke: int): 513 x0 = int(width * 0.24) 514 y0 = int(height * 0.82) 515 x1 = int(width * 0.76) 516 y1 = int(height * 0.90) 517 draw.rounded_rectangle((x0, y0, x1, y1), radius=10, outline="white", width=stroke) 518 segments = 12 519 gap = 4 520 seg_w = (x1 - x0 - 40 - gap * (segments - 1)) // segments 521 sx = x0 + 20 522 for _ in range(segments): 523 draw.rectangle((sx, y0 + 20, sx + seg_w, y1 - 20), outline="white", width=max(1, stroke // 2)) 524 sx += seg_w + gap 525 526 527def draw_dev(draw: ImageDraw.ImageDraw, x: int, base_y: int, scale: float, state: str, stroke: int): 528 if state == "collapsed": 529 head_r = int(38 * scale) 530 head_x = x - int(52 * scale) 531 head_y = base_y - int(35 * scale) 532 draw.ellipse((head_x - head_r, head_y - head_r, head_x + head_r, head_y + head_r), outline="white", width=stroke) 533 draw.rectangle((head_x - int(30 * scale), head_y - int(10 * scale), head_x - int(4 * scale), head_y + int(8 * scale)), outline="white", width=max(1, stroke // 2)) 534 draw.rectangle((head_x + int(4 * scale), head_y - int(10 * scale), head_x + int(30 * scale), head_y + int(8 * scale)), outline="white", width=max(1, stroke // 2)) 535 draw.line((head_x + head_r, head_y, x + int(95 * scale), base_y), fill="white", width=stroke) 536 draw.line((x + int(20 * scale), base_y, x + int(110 * scale), base_y + int(35 * scale)), fill="white", width=stroke) 537 draw.line((x + int(20 * scale), base_y, x + int(110 * scale), base_y - int(35 * scale)), fill="white", width=stroke) 538 return 539 540 seated = state == "seated" 541 head_r = int(38 * scale) 542 head_y = base_y - int(170 * scale if seated else 210 * scale) 543 draw.ellipse((x - head_r, head_y - head_r, x + head_r, head_y + head_r), outline="white", width=stroke) 544 draw.rectangle((x - int(34 * scale), head_y - int(10 * scale), x - int(5 * scale), head_y + int(9 * scale)), outline="white", width=max(1, stroke // 2)) 545 draw.rectangle((x + int(5 * scale), head_y - int(10 * scale), x + int(34 * scale), head_y + int(9 * scale)), outline="white", width=max(1, stroke // 2)) 546 draw.line((x - int(5 * scale), head_y, x + int(5 * scale), head_y), fill="white", width=max(1, stroke // 2)) 547 548 neck_y = head_y + head_r 549 hip_y = base_y - int(62 * scale if seated else 70 * scale) 550 draw.line((x, neck_y, x, hip_y), fill="white", width=stroke) 551 552 if state == "excited": 553 draw.line((x, neck_y + int(18 * scale), x - int(58 * scale), neck_y - int(50 * scale)), fill="white", width=stroke) 554 draw.line((x, neck_y + int(18 * scale), x + int(58 * scale), neck_y - int(50 * scale)), fill="white", width=stroke) 555 elif seated: 556 draw.line((x, neck_y + int(25 * scale), x + int(80 * scale), neck_y + int(60 * scale)), fill="white", width=stroke) 557 draw.line((x, neck_y + int(25 * scale), x + int(58 * scale), neck_y + int(82 * scale)), fill="white", width=stroke) 558 else: 559 draw.line((x, neck_y + int(25 * scale), x - int(45 * scale), neck_y + int(88 * scale)), fill="white", width=stroke) 560 draw.line((x, neck_y + int(25 * scale), x + int(45 * scale), neck_y + int(88 * scale)), fill="white", width=stroke) 561 562 if seated: 563 draw.line((x, hip_y, x + int(70 * scale), base_y - int(34 * scale)), fill="white", width=stroke) 564 draw.line((x + int(70 * scale), base_y - int(34 * scale), x + int(105 * scale), base_y - int(34 * scale)), fill="white", width=stroke) 565 draw.line((x, hip_y, x - int(15 * scale), base_y), fill="white", width=stroke) 566 draw.line((x - int(50 * scale), base_y, x + int(35 * scale), base_y), fill="white", width=stroke) 567 else: 568 draw.line((x, hip_y, x - int(42 * scale), base_y), fill="white", width=stroke) 569 draw.line((x, hip_y, x + int(42 * scale), base_y), fill="white", width=stroke) 570 draw.line((x - int(42 * scale), base_y, x - int(65 * scale), base_y), fill="white", width=stroke) 571 draw.line((x + int(42 * scale), base_y, x + int(65 * scale), base_y), fill="white", width=stroke) 572 573 574def draw_clanker(draw: ImageDraw.ImageDraw, x: int, base_y: int, scale: float, state: str, stroke: int): 575 head_w = int(110 * scale) 576 head_h = int(82 * scale) 577 head_y0 = base_y - int(255 * scale) 578 head_y1 = head_y0 + head_h 579 draw.rounded_rectangle((x - head_w // 2, head_y0, x + head_w // 2, head_y1), radius=int(10 * scale), outline="white", width=stroke) 580 draw.rectangle((x - int(28 * scale), head_y0 + int(30 * scale), x - int(14 * scale), head_y0 + int(55 * scale)), outline="white", width=stroke) 581 draw.rectangle((x + int(14 * scale), head_y0 + int(30 * scale), x + int(28 * scale), head_y0 + int(55 * scale)), outline="white", width=stroke) 582 if state == "worried": 583 draw.arc((x - int(20 * scale), head_y0 + int(54 * scale), x + int(20 * scale), head_y0 + int(82 * scale)), 190, 350, fill="white", width=stroke) 584 else: 585 draw.line((x - int(20 * scale), head_y0 + int(68 * scale), x + int(20 * scale), head_y0 + int(68 * scale)), fill="white", width=stroke) 586 draw.line((x, head_y0, x, head_y0 - int(42 * scale)), fill="white", width=stroke) 587 draw.ellipse((x - int(8 * scale), head_y0 - int(58 * scale), x + int(8 * scale), head_y0 - int(42 * scale)), outline="white", width=stroke) 588 589 body_y0 = head_y1 + int(18 * scale) 590 body_y1 = body_y0 + int(90 * scale) 591 draw.rounded_rectangle((x - int(42 * scale), body_y0, x + int(42 * scale), body_y1), radius=int(8 * scale), outline="white", width=stroke) 592 draw.line((x - int(20 * scale), body_y0 + int(30 * scale), x + int(20 * scale), body_y0 + int(30 * scale)), fill="white", width=max(1, stroke // 2)) 593 draw.line((x - int(20 * scale), body_y0 + int(48 * scale), x + int(12 * scale), body_y0 + int(48 * scale)), fill="white", width=max(1, stroke // 2)) 594 595 shoulder_y = body_y0 + int(22 * scale) 596 hand_y = body_y1 - int(10 * scale) 597 if state == "excited": 598 draw.line((x - int(42 * scale), shoulder_y, x - int(88 * scale), shoulder_y - int(70 * scale)), fill="white", width=stroke) 599 draw.line((x + int(42 * scale), shoulder_y, x + int(88 * scale), shoulder_y - int(70 * scale)), fill="white", width=stroke) 600 else: 601 draw.line((x - int(42 * scale), shoulder_y, x - int(82 * scale), hand_y), fill="white", width=stroke) 602 draw.line((x + int(42 * scale), shoulder_y, x + int(82 * scale), hand_y), fill="white", width=stroke) 603 draw.arc((x - int(98 * scale), hand_y - int(18 * scale), x - int(66 * scale), hand_y + int(18 * scale)), 80, 290, fill="white", width=stroke) 604 draw.arc((x + int(66 * scale), hand_y - int(18 * scale), x + int(98 * scale), hand_y + int(18 * scale)), -110, 100, fill="white", width=stroke) 605 606 draw.line((x - int(22 * scale), body_y1, x - int(35 * scale), base_y), fill="white", width=stroke) 607 draw.line((x + int(22 * scale), body_y1, x + int(35 * scale), base_y), fill="white", width=stroke) 608 draw.rectangle((x - int(55 * scale), base_y - int(16 * scale), x - int(22 * scale), base_y), outline="white", width=stroke) 609 draw.rectangle((x + int(22 * scale), base_y - int(16 * scale), x + int(55 * scale), base_y), outline="white", width=stroke) 610 611 612def draw_sparks(draw: ImageDraw.ImageDraw, width: int, height: int, stroke: int): 613 for x, y in [(0.45, 0.42), (0.56, 0.36), (0.63, 0.48), (0.37, 0.52)]: 614 cx, cy = int(width * x), int(height * y) 615 draw.line((cx - 18, cy + 22, cx, cy - 22, cx + 18, cy + 10), fill="white", width=stroke) 616 617 618def dev_control_sprite_name(text: str) -> str: 619 if _has_any(text, ("dead", "x-eye", "x eye")): 620 return "dev_pose_dead" 621 if _has_any(text, ("collapsed", "floor", "lies flat", "flat on", "burnout")): 622 return "dev_pose_collapsed" 623 if _has_any(text, ("slumped", "exhausted", "tired")): 624 return "dev_pose_slumped" 625 if _has_any(text, ("panic", "panicked", "overwhelmed", "holy", "chaos")): 626 return "dev_pose_overwhelmed" 627 if _has_any(text, ("resurrected", "zombie")): 628 return "dev_pose_zombie" 629 if _has_any(text, ("seated", "typing", "laptop", "terminal", "nvim", "code")): 630 return "dev_seated_laptop" 631 if _has_any(text, ("excited", "wow", "arms up", "throws both arms", "celebrat")): 632 return "dev_excited_body" 633 if _has_any(text, ("coffee", "mug")): 634 return "dev_mug_body" 635 if _has_any(text, ("running", "run")): 636 return "dev_running_body" 637 return "dev_neutral_body" 638 639 640def clanker_control_sprite_name(text: str) -> str: 641 if _has_any(text, ("laptop", "typing", "interfacing")) and "clanker" in text and "dev" not in text: 642 return "clanker_at_laptop" 643 if _has_any(text, ("happy", "celebrat", "triumphant", "hyped", "success", "proud", "confident")): 644 return "clanker_happy_body" 645 return "clanker_neutral_body" 646 647 648def draw_reference_characters( 649 image: Image.Image, 650 project_root: Path, 651 text: str, 652 dev_x: float, 653 clanker_x: float, 654 base_y: int, 655 scale: float, 656) -> bool: 657 dev_name = dev_control_sprite_name(text) 658 clanker_name = clanker_control_sprite_name(text) 659 dev_sprite = control_reference_sprite(project_root, dev_name) 660 clanker_sprite = control_reference_sprite(project_root, clanker_name) 661 if dev_sprite is None or clanker_sprite is None: 662 return False 663 664 shot_scale = 1.0 665 if "close" in text: 666 shot_scale = 1.18 667 elif "wide" in text: 668 shot_scale = 0.88 669 670 dev_height = int((300 if dev_name.startswith("dev_pose_") else 330) * scale * shot_scale) 671 clanker_height = int(335 * scale * shot_scale) 672 dev_center = dev_x 673 if dev_name in {"dev_seated_laptop", "dev_laptop_body"}: 674 # These sprites include the desk/laptop, so their visual center needs to 675 # sit closer to the middle of the panel than the Dev's head alone. 676 dev_center = min(0.48, dev_x + 0.08) 677 dev_height = int(300 * scale * shot_scale) 678 if dev_name in {"dev_pose_collapsed", "dev_pose_dead"}: 679 dev_center = max(0.28, dev_x + 0.05) 680 dev_height = int(210 * scale * shot_scale) 681 682 paste_control_sprite(image, dev_sprite, int(image.width * dev_center), base_y, dev_height) 683 paste_control_sprite(image, clanker_sprite, int(image.width * clanker_x), base_y, clanker_height) 684 return True 685 686 687def make_layout_control_image(panel: PanelSpec, width: int, height: int, project_root: Path | None = None) -> Image.Image: 688 text = f"{panel.shot} {_panel_text(panel)}" 689 composition = (panel.composition or "").lower() 690 scale = min(width / 1024, height / 768) 691 stroke = max(2, int(4 * scale)) 692 image = Image.new("RGBA", (width, height), (0, 0, 0, 255)) 693 draw = ImageDraw.Draw(image) 694 695 dev_x = _clause_position(composition, "dev", 0.31) 696 clanker_x = _clause_position(composition, "clanker", 0.69) 697 if abs(dev_x - clanker_x) < 0.18: 698 dev_x, clanker_x = 0.31, 0.69 699 700 foreground = "foreground" in text 701 base_y = int(height * (0.86 if foreground else 0.80)) 702 703 dev_sprite_name = dev_control_sprite_name(text) 704 sprite_has_laptop = dev_sprite_name in {"dev_seated_laptop", "dev_laptop_body", "dev_pose_slumped", "dev_pose_overwhelmed"} 705 if _has_any(text, ("laptop", "terminal", "monitor", "nvim", "code")) and not sprite_has_laptop: 706 draw_laptop(draw, width, height, width * 0.50, height * (0.68 if foreground else 0.70), scale, stroke) 707 if "token meter" in text or "tokens" in text: 708 draw_token_meter(draw, width, height, stroke) 709 710 used_reference_sprites = False 711 if project_root is not None: 712 used_reference_sprites = draw_reference_characters(image, project_root, text, dev_x, clanker_x, base_y, scale) 713 714 if not used_reference_sprites: 715 dev_state = "standing" 716 if _has_any(text, ("seated", "typing", "laptop")): 717 dev_state = "seated" 718 if _has_any(text, ("excited", "wow", "arms up", "throws both arms", "celebrat")): 719 dev_state = "excited" 720 if _has_any(text, ("collapsed", "floor", "dead", "lies flat", "flat on")): 721 dev_state = "collapsed" 722 723 clanker_state = "standing" 724 if _has_any(text, ("arms raised", "celebrat", "triumphant", "hyped")): 725 clanker_state = "excited" 726 if _has_any(text, ("concerned", "worried")): 727 clanker_state = "worried" 728 729 draw_dev(draw, int(width * dev_x), base_y, scale, dev_state, stroke) 730 draw_clanker(draw, int(width * clanker_x), base_y, scale, clanker_state, stroke) 731 732 if _has_any(text, ("spark", "chaos", "overclock", "erupts", "debris")): 733 draw_sparks(draw, width, height, stroke) 734 735 return image.convert("RGB") 736 737 738def fit_control_image(image: Image.Image, width: int, height: int) -> Image.Image: 739 image = ImageOps.exif_transpose(image).convert("RGB") 740 image = ImageOps.contain(image, (width, height), method=Image.Resampling.LANCZOS) 741 canvas = Image.new("RGB", (width, height), "black") 742 canvas.paste(image, ((width - image.width) // 2, (height - image.height) // 2)) 743 return canvas 744 745 746def canny_control_image(image: Image.Image) -> Image.Image: 747 import cv2 748 import numpy as np 749 750 gray = np.array(ImageOps.grayscale(image)) 751 edges = cv2.Canny(gray, 80, 180) 752 return Image.fromarray(edges).convert("RGB") 753 754 755def make_control_image( 756 config: GenerationConfig, 757 panel: PanelSpec, 758 project_root: Path, 759 output_path: Path, 760 panel_number: int, 761) -> Image.Image | None: 762 if not config.enableControlNet: 763 return None 764 765 if panel.controlImage: 766 control_path = (project_root / panel.controlImage).resolve() 767 if not control_path.exists(): 768 raise FileNotFoundError(f"Control image not found: {control_path}") 769 image = fit_control_image(Image.open(control_path), config.width, config.height) 770 if config.controlGuideMode == "canny": 771 image = canny_control_image(image) 772 else: 773 image = make_layout_control_image(panel, config.width, config.height, project_root) 774 775 debug_path = output_path.parent / f"control-panel-{panel_number:02d}.png" 776 debug_path.parent.mkdir(parents=True, exist_ok=True) 777 image.save(debug_path) 778 return image 779 780 781def generate_panel_art( 782 comic: ComicSpec, 783 panel: PanelSpec, 784 panel_number: int, 785 project_root: Path, 786 output_path: Path, 787) -> tuple[Path, int]: 788 config = comic.generation 789 pipe = load_pipeline(config) 790 791 import torch 792 793 if hasattr(pipe, "maybe_free_model_hooks"): 794 pipe.maybe_free_model_hooks() 795 if config.device == "cuda": 796 torch.cuda.empty_cache() 797 798 reference_images = load_reference_images(config, panel, project_root) 799 execution_device = getattr(pipe, "_execution_device", config.device) 800 do_classifier_free_guidance = config.guidanceScale > 1.0 801 ip_adapter_image_embeds = make_ip_adapter_image_embeds( 802 pipe, 803 reference_images, 804 torch, 805 execution_device, 806 do_classifier_free_guidance, 807 ) 808 809 seed = panel.seed if panel.seed is not None else config.seed 810 if seed is None: 811 seed = random.SystemRandom().randint(0, 2**32 - 1) 812 generator = torch.Generator(device=config.device).manual_seed(seed) 813 814 negative_prompt = config.negativePrompt 815 if panel.negativePrompt: 816 negative_prompt = f"{negative_prompt}, {panel.negativePrompt}" 817 818 call_args = { 819 "prompt": build_prompt(comic, panel, panel_number), 820 "negative_prompt": negative_prompt, 821 "width": config.width, 822 "height": config.height, 823 "num_inference_steps": config.steps, 824 "guidance_scale": config.guidanceScale, 825 "generator": generator, 826 } 827 if ip_adapter_image_embeds is not None: 828 call_args["ip_adapter_image_embeds"] = ip_adapter_image_embeds 829 else: 830 call_args["ip_adapter_image"] = make_reference_contact_sheet(reference_images) 831 832 if config.device == "cuda": 833 torch.cuda.empty_cache() 834 835 control_image = make_control_image(config, panel, project_root, output_path, panel_number) 836 if control_image is not None: 837 call_args.update( 838 { 839 "image": control_image, 840 "controlnet_conditioning_scale": panel.controlNetScale 841 if panel.controlNetScale is not None 842 else config.controlNetScale, 843 "control_guidance_start": config.controlNetStart, 844 "control_guidance_end": config.controlNetEnd, 845 } 846 ) 847 848 image = pipe(**call_args).images[0] 849 850 if hasattr(pipe, "maybe_free_model_hooks"): 851 pipe.maybe_free_model_hooks() 852 if config.device == "cuda": 853 torch.cuda.empty_cache() 854 855 output_path.parent.mkdir(parents=True, exist_ok=True) 856 ImageOps.exif_transpose(image).save(output_path) 857 return output_path, seed