7517
Cd
clanker-and-the-dev / files
clanker-and-the-dev / .agents / skills / add-new-comic / scripts / composite_panel.py
13.1 KB
1from __future__ import annotations 2 3import re 4import textwrap 5from pathlib import Path 6 7from PIL import Image, ImageChops, ImageDraw, ImageFont, ImageFilter, ImageOps 8 9from models import DialogueLine, PanelSpec 10 11GREEN = "#00ff66" 12LIGHT_GREEN = "#7cff8a" 13DARK = "#020806" 14BLACK_ALPHA = (0, 0, 0, 205) 15 16 17def clean_markup(text: str) -> str: 18 return re.sub(r"[*_`]+", "", text).strip() 19 20 21def load_font(project_root: Path, size: int) -> ImageFont.FreeTypeFont | ImageFont.ImageFont: 22 font_path = project_root / "assets" / "fonts" / "VT323-Regular.ttf" 23 if font_path.exists(): 24 return ImageFont.truetype(str(font_path), size=size) 25 return ImageFont.load_default() 26 27 28def terminal_line_art(raw: Image.Image, threshold: int = 95, stroke: int = 3) -> Image.Image: 29 """Convert soft SDXL output into the crisp green-on-black reference style.""" 30 raw = ImageOps.exif_transpose(raw).convert("RGB") 31 try: 32 import cv2 33 import numpy as np 34 35 gray = np.array(ImageOps.autocontrast(ImageOps.grayscale(raw), cutoff=1)) 36 mask = (gray > threshold).astype(np.uint8) * 255 37 mask = cv2.medianBlur(mask, 3) 38 39 height, width = mask.shape 40 # Panel specs reserve the upper band for compositor-owned bubbles. 41 # Clearing it removes common SDXL hallucinations copied from reference 42 # sheet headers/frames without touching the staged characters. 43 mask[: int(height * 0.28), :] = 0 44 clean_mask = np.zeros_like(mask) 45 keep_filled = np.zeros_like(mask) 46 label_count, labels, stats, _ = cv2.connectedComponentsWithStats(mask, connectivity=8) 47 for label in range(1, label_count): 48 x, y, w, h, area = stats[label] 49 if area < 8: 50 continue 51 # SDXL sometimes invents a full panel/grid because the reference 52 # sheets contain panels. Drop huge border components; our compositor 53 # draws the real frame afterward. 54 if w > width * 0.82 and h > height * 0.58: 55 continue 56 # Reject thin generated header/footer bars copied from reference 57 # sheets, screenshots, or UI mockups. Real comic lettering is drawn 58 # later by the compositor. 59 if w > width * 0.45 and h < height * 0.06 and (y < height * 0.18 or y + h > height * 0.82): 60 continue 61 if h > height * 0.46 and w < width * 0.18: 62 continue 63 if y < height * 0.50 and y + h < height * 0.66 and w > width * 0.16 and h > height * 0.15: 64 continue 65 component = labels == label 66 clean_mask[component] = 255 67 # Preserve small filled UI/robot-eye shapes instead of skeletonizing 68 # them into X marks. 69 if area <= 1200 and w <= 52 and h <= 52: 70 keep_filled[component] = 255 71 mask = clean_mask 72 73 skeleton = np.zeros(mask.shape, np.uint8) 74 element = cv2.getStructuringElement(cv2.MORPH_CROSS, (3, 3)) 75 working = mask.copy() 76 while cv2.countNonZero(working): 77 eroded = cv2.erode(working, element) 78 opened = cv2.dilate(eroded, element) 79 skeleton = cv2.bitwise_or(skeleton, cv2.subtract(working, opened)) 80 working = eroded 81 82 line_mask = cv2.bitwise_or(skeleton, keep_filled) 83 if stroke > 1: 84 kernel = np.ones((stroke, stroke), np.uint8) 85 line_mask = cv2.dilate(line_mask, kernel, iterations=1) 86 87 canvas = np.zeros((*line_mask.shape, 3), dtype=np.uint8) 88 canvas[:] = (2, 8, 6) 89 canvas[line_mask > 0] = (0, 255, 102) 90 return Image.fromarray(canvas, "RGB") 91 except Exception: 92 gray = ImageOps.autocontrast(ImageOps.grayscale(raw), cutoff=1) 93 mask = gray.point(lambda p: 255 if p > threshold else 0) 94 if stroke > 1: 95 mask = mask.filter(ImageFilter.MaxFilter(stroke if stroke % 2 else stroke + 1)) 96 return Image.composite(Image.new("RGB", raw.size, GREEN), Image.new("RGB", raw.size, DARK), mask) 97 98 99def prepare_art( 100 raw: Image.Image, 101 mode: str = "terminal-lineart", 102 threshold: int = 95, 103 stroke: int = 3, 104) -> Image.Image: 105 if mode == "none": 106 return ImageOps.exif_transpose(raw).convert("RGB") 107 if mode == "monochrome": 108 return ImageOps.colorize(ImageOps.grayscale(raw), black=DARK, white=GREEN) 109 return terminal_line_art(raw, threshold=threshold, stroke=stroke) 110 111 112def control_guide_line_art(control_path: Path, size: tuple[int, int]) -> Image.Image | None: 113 if not control_path.exists(): 114 return None 115 guide = Image.open(control_path).convert("L") 116 if guide.size != size: 117 guide = guide.resize(size, Image.Resampling.LANCZOS) 118 mask = guide.point(lambda p: 255 if p > 24 else 0) 119 if not mask.getbbox(): 120 return None 121 return Image.composite(Image.new("RGB", size, GREEN), Image.new("RGB", size, DARK), mask) 122 123 124def control_guide_mask(control_path: Path, size: tuple[int, int]) -> Image.Image | None: 125 if not control_path.exists(): 126 return None 127 guide = Image.open(control_path).convert("L") 128 if guide.size != size: 129 guide = guide.resize(size, Image.Resampling.LANCZOS) 130 mask = guide.point(lambda p: 255 if p > 24 else 0) 131 if not mask.getbbox(): 132 return None 133 return mask 134 135 136def line_mask_from_terminal_art(art: Image.Image) -> Image.Image: 137 gray = ImageOps.grayscale(art.convert("RGB")) 138 return gray.point(lambda p: 255 if p > 28 else 0) 139 140 141def mask_to_terminal_art(mask: Image.Image, size: tuple[int, int]) -> Image.Image: 142 if mask.size != size: 143 mask = mask.resize(size, Image.Resampling.NEAREST) 144 return Image.composite(Image.new("RGB", size, GREEN), Image.new("RGB", size, DARK), mask) 145 146 147def filter_line_art_to_control_guide(art: Image.Image, guide_mask: Image.Image, padding: int = 0) -> Image.Image: 148 """Keep generated strokes only when they support the reference-derived guide. 149 150 SDXL often creates faint CRT bands, window/card rectangles, and floating blobs 151 because the references are terminal-style screenshots. The guide already has 152 the desired character/prop sketch, so generated strokes are treated as local 153 embellishment and are clipped back to the guide before the guide is merged. 154 """ 155 try: 156 import cv2 157 import numpy as np 158 159 raw_mask = np.array(line_mask_from_terminal_art(art), dtype=np.uint8) 160 guide = np.array(guide_mask, dtype=np.uint8) 161 kernel_size = max(1, padding * 2 + 1) 162 kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (kernel_size, kernel_size)) 163 keep_zone = cv2.dilate(guide, kernel, iterations=1) 164 clipped = cv2.bitwise_and(raw_mask, keep_zone) 165 166 cleaned = np.zeros_like(clipped) 167 label_count, labels, stats, _ = cv2.connectedComponentsWithStats(clipped, connectivity=8) 168 for label in range(1, label_count): 169 x, y, w, h, area = stats[label] 170 if area < 8: 171 continue 172 component = labels == label 173 density = area / max(1, w * h) 174 175 # Filled SDXL blobs make the final panels look less like the clean 176 # reference sketches. The exact guide is merged later, so dense raw 177 # fills can be rejected even when they overlap a desired guide mark. 178 if density > 0.34 and area > 24: 179 continue 180 if w > raw_mask.shape[1] * 0.38 and h < raw_mask.shape[0] * 0.04: 181 continue 182 cleaned[component] = 255 183 184 return mask_to_terminal_art(Image.fromarray(cleaned, "L"), art.size) 185 except Exception: 186 clipped = ImageChops.multiply(line_mask_from_terminal_art(art), guide_mask.filter(ImageFilter.MaxFilter(padding | 1))) 187 return mask_to_terminal_art(clipped, art.size) 188 189 190def merge_control_guide(art: Image.Image, raw_art_path: Path, panel_number: int) -> Image.Image: 191 control_path = raw_art_path.with_name(f"control-panel-{panel_number:02d}.png") 192 guide = control_guide_line_art(control_path, art.size) 193 guide_mask = control_guide_mask(control_path, art.size) 194 if guide is None or guide_mask is None: 195 return art 196 # The ControlNet guide is derived from art-reference sprites. Keeping it in 197 # the final line art repairs SDXL drift. Generated strokes are now kept only 198 # to that guide, which removes scanlines/floating artifacts while retaining 199 # the cleaned SDXL panel generation as the raw/debug artifact. 200 filtered = filter_line_art_to_control_guide(art.convert("RGB"), guide_mask) 201 return ImageChops.lighter(filtered, guide) 202 203 204def wrap_for_width(draw: ImageDraw.ImageDraw, text: str, font: ImageFont.ImageFont, max_width: int) -> list[str]: 205 text = clean_markup(text) 206 words = text.split() 207 lines: list[str] = [] 208 current: list[str] = [] 209 for word in words: 210 candidate = " ".join([*current, word]) 211 if draw.textlength(candidate, font=font) <= max_width or not current: 212 current.append(word) 213 else: 214 lines.append(" ".join(current)) 215 current = [word] 216 if current: 217 lines.append(" ".join(current)) 218 return lines or [""] 219 220 221def draw_tail(draw: ImageDraw.ImageDraw, box: tuple[int, int, int, int], speaker: str, outline: str): 222 x0, y0, x1, y1 = box 223 if speaker == "dev": 224 points = [(x0 + 80, y1 - 2), (x0 + 105, y1 + 34), (x0 + 145, y1 - 2)] 225 elif speaker == "clanker": 226 points = [(x1 - 145, y1 - 2), (x1 - 105, y1 + 34), (x1 - 80, y1 - 2)] 227 else: 228 return 229 draw.polygon(points, fill=BLACK_ALPHA, outline=outline) 230 231 232def draw_bubble( 233 overlay: Image.Image, 234 text: str, 235 speaker: str, 236 box: tuple[int, int, int, int], 237 font: ImageFont.ImageFont, 238): 239 draw = ImageDraw.Draw(overlay) 240 x0, y0, x1, y1 = box 241 draw.rounded_rectangle(box, radius=18, fill=BLACK_ALPHA, outline=GREEN, width=4) 242 draw_tail(draw, box, speaker, GREEN) 243 244 label = "" if speaker == "narrator" else f"{speaker.upper()}: " 245 lines = wrap_for_width(draw, f"{label}{text}", font, (x1 - x0) - 36) 246 line_height = int(font.size * 0.9) if hasattr(font, "size") else 24 247 total_height = len(lines) * line_height 248 y = y0 + max(12, ((y1 - y0) - total_height) // 2) 249 for line in lines: 250 draw.text((x0 + 18, y), line, font=font, fill=LIGHT_GREEN) 251 y += line_height 252 253 254def bubble_box(index: int, total: int, speaker: str, image_width: int) -> tuple[int, int, int, int]: 255 top = 28 + index * 116 256 height = 92 257 margin = 34 258 if speaker == "dev": 259 return (margin, top, int(image_width * 0.62), top + height) 260 if speaker == "clanker": 261 return (int(image_width * 0.38), top, image_width - margin, top + height) 262 return (margin, top, image_width - margin, top + height) 263 264 265def draw_terminal( 266 overlay: Image.Image, 267 lines: list[str], 268 project_root: Path, 269): 270 if not lines: 271 return 272 draw = ImageDraw.Draw(overlay) 273 width, height = overlay.size 274 font = load_font(project_root, 34) 275 clean_lines = [clean_markup(line) for line in lines[:4]] 276 line_height = 35 277 x = 56 278 y = height - (len(clean_lines) * line_height) - 48 279 # Render only the joke's code/output text. Avoid terminal-window chrome, 280 # title bars, boxed footers, or UI panels inside the comic image. 281 for line in clean_lines: 282 shortened = textwrap.shorten(line, width=42, placeholder="…") 283 draw.text( 284 (x, y), 285 shortened, 286 font=font, 287 fill=LIGHT_GREEN, 288 stroke_width=2, 289 stroke_fill=(0, 0, 0, 230), 290 ) 291 y += line_height 292 293 294def composite_panel( 295 raw_art_path: Path, 296 output_path: Path, 297 panel: PanelSpec, 298 project_root: Path, 299 panel_number: int, 300 art_postprocess_mode: str = "terminal-lineart", 301 line_art_threshold: int = 95, 302 line_art_stroke: int = 3, 303) -> Path: 304 raw = Image.open(raw_art_path).convert("RGB") 305 base_art = prepare_art( 306 raw, 307 mode=art_postprocess_mode, 308 threshold=line_art_threshold, 309 stroke=line_art_stroke, 310 ) 311 if art_postprocess_mode == "terminal-lineart": 312 base_art = merge_control_guide(base_art, raw_art_path, panel_number) 313 base = base_art.convert("RGBA") 314 overlay = Image.new("RGBA", base.size, (0, 0, 0, 0)) 315 draw = ImageDraw.Draw(overlay) 316 width, height = base.size 317 318 # Simple comic panel frame. No baked-in panel number, title card, header, 319 # footer, or site chrome; the page provides that in HTML. 320 draw.rectangle((8, 8, width - 8, height - 8), outline=GREEN, width=3) 321 322 bubble_font = load_font(project_root, 42) 323 items: list[DialogueLine | tuple[str, str]] = [] 324 if panel.caption: 325 items.append(("narrator", panel.caption)) 326 items.extend(panel.dialogue) 327 328 for index, item in enumerate(items[:3]): 329 if isinstance(item, tuple): 330 speaker, text = item 331 else: 332 speaker, text = item.speaker, item.text 333 draw_bubble(overlay, text, speaker, bubble_box(index, len(items), speaker, width), bubble_font) 334 335 draw_terminal(overlay, panel.terminalLines, project_root) 336 337 composed = Image.alpha_composite(base, overlay).convert("RGB") 338 output_path.parent.mkdir(parents=True, exist_ok=True) 339 composed.save(output_path, quality=95) 340 return output_path