xonecas.io
7517
Cd

clanker-and-the-dev / files

clanker-and-the-dev / .agents / skills / add-new-comic / scripts / composite_panel.py

13.1 KB
1from __future__ import annotations
2
3import re
4import textwrap
5from pathlib import Path
6
7from PIL import Image, ImageChops, ImageDraw, ImageFont, ImageFilter, ImageOps
8
9from models import DialogueLine, PanelSpec
10
11GREEN = "#00ff66"
12LIGHT_GREEN = "#7cff8a"
13DARK = "#020806"
14BLACK_ALPHA = (0, 0, 0, 205)
15
16
17def clean_markup(text: str) -> str:
18    return re.sub(r"[*_`]+", "", text).strip()
19
20
21def load_font(project_root: Path, size: int) -> ImageFont.FreeTypeFont | ImageFont.ImageFont:
22    font_path = project_root / "assets" / "fonts" / "VT323-Regular.ttf"
23    if font_path.exists():
24        return ImageFont.truetype(str(font_path), size=size)
25    return ImageFont.load_default()
26
27
28def terminal_line_art(raw: Image.Image, threshold: int = 95, stroke: int = 3) -> Image.Image:
29    """Convert soft SDXL output into the crisp green-on-black reference style."""
30    raw = ImageOps.exif_transpose(raw).convert("RGB")
31    try:
32        import cv2
33        import numpy as np
34
35        gray = np.array(ImageOps.autocontrast(ImageOps.grayscale(raw), cutoff=1))
36        mask = (gray > threshold).astype(np.uint8) * 255
37        mask = cv2.medianBlur(mask, 3)
38
39        height, width = mask.shape
40        # Panel specs reserve the upper band for compositor-owned bubbles.
41        # Clearing it removes common SDXL hallucinations copied from reference
42        # sheet headers/frames without touching the staged characters.
43        mask[: int(height * 0.28), :] = 0
44        clean_mask = np.zeros_like(mask)
45        keep_filled = np.zeros_like(mask)
46        label_count, labels, stats, _ = cv2.connectedComponentsWithStats(mask, connectivity=8)
47        for label in range(1, label_count):
48            x, y, w, h, area = stats[label]
49            if area < 8:
50                continue
51            # SDXL sometimes invents a full panel/grid because the reference
52            # sheets contain panels. Drop huge border components; our compositor
53            # draws the real frame afterward.
54            if w > width * 0.82 and h > height * 0.58:
55                continue
56            # Reject thin generated header/footer bars copied from reference
57            # sheets, screenshots, or UI mockups. Real comic lettering is drawn
58            # later by the compositor.
59            if w > width * 0.45 and h < height * 0.06 and (y < height * 0.18 or y + h > height * 0.82):
60                continue
61            if h > height * 0.46 and w < width * 0.18:
62                continue
63            if y < height * 0.50 and y + h < height * 0.66 and w > width * 0.16 and h > height * 0.15:
64                continue
65            component = labels == label
66            clean_mask[component] = 255
67            # Preserve small filled UI/robot-eye shapes instead of skeletonizing
68            # them into X marks.
69            if area <= 1200 and w <= 52 and h <= 52:
70                keep_filled[component] = 255
71        mask = clean_mask
72
73        skeleton = np.zeros(mask.shape, np.uint8)
74        element = cv2.getStructuringElement(cv2.MORPH_CROSS, (3, 3))
75        working = mask.copy()
76        while cv2.countNonZero(working):
77            eroded = cv2.erode(working, element)
78            opened = cv2.dilate(eroded, element)
79            skeleton = cv2.bitwise_or(skeleton, cv2.subtract(working, opened))
80            working = eroded
81
82        line_mask = cv2.bitwise_or(skeleton, keep_filled)
83        if stroke > 1:
84            kernel = np.ones((stroke, stroke), np.uint8)
85            line_mask = cv2.dilate(line_mask, kernel, iterations=1)
86
87        canvas = np.zeros((*line_mask.shape, 3), dtype=np.uint8)
88        canvas[:] = (2, 8, 6)
89        canvas[line_mask > 0] = (0, 255, 102)
90        return Image.fromarray(canvas, "RGB")
91    except Exception:
92        gray = ImageOps.autocontrast(ImageOps.grayscale(raw), cutoff=1)
93        mask = gray.point(lambda p: 255 if p > threshold else 0)
94        if stroke > 1:
95            mask = mask.filter(ImageFilter.MaxFilter(stroke if stroke % 2 else stroke + 1))
96        return Image.composite(Image.new("RGB", raw.size, GREEN), Image.new("RGB", raw.size, DARK), mask)
97
98
99def prepare_art(
100    raw: Image.Image,
101    mode: str = "terminal-lineart",
102    threshold: int = 95,
103    stroke: int = 3,
104) -> Image.Image:
105    if mode == "none":
106        return ImageOps.exif_transpose(raw).convert("RGB")
107    if mode == "monochrome":
108        return ImageOps.colorize(ImageOps.grayscale(raw), black=DARK, white=GREEN)
109    return terminal_line_art(raw, threshold=threshold, stroke=stroke)
110
111
112def control_guide_line_art(control_path: Path, size: tuple[int, int]) -> Image.Image | None:
113    if not control_path.exists():
114        return None
115    guide = Image.open(control_path).convert("L")
116    if guide.size != size:
117        guide = guide.resize(size, Image.Resampling.LANCZOS)
118    mask = guide.point(lambda p: 255 if p > 24 else 0)
119    if not mask.getbbox():
120        return None
121    return Image.composite(Image.new("RGB", size, GREEN), Image.new("RGB", size, DARK), mask)
122
123
124def control_guide_mask(control_path: Path, size: tuple[int, int]) -> Image.Image | None:
125    if not control_path.exists():
126        return None
127    guide = Image.open(control_path).convert("L")
128    if guide.size != size:
129        guide = guide.resize(size, Image.Resampling.LANCZOS)
130    mask = guide.point(lambda p: 255 if p > 24 else 0)
131    if not mask.getbbox():
132        return None
133    return mask
134
135
136def line_mask_from_terminal_art(art: Image.Image) -> Image.Image:
137    gray = ImageOps.grayscale(art.convert("RGB"))
138    return gray.point(lambda p: 255 if p > 28 else 0)
139
140
141def mask_to_terminal_art(mask: Image.Image, size: tuple[int, int]) -> Image.Image:
142    if mask.size != size:
143        mask = mask.resize(size, Image.Resampling.NEAREST)
144    return Image.composite(Image.new("RGB", size, GREEN), Image.new("RGB", size, DARK), mask)
145
146
147def filter_line_art_to_control_guide(art: Image.Image, guide_mask: Image.Image, padding: int = 0) -> Image.Image:
148    """Keep generated strokes only when they support the reference-derived guide.
149
150    SDXL often creates faint CRT bands, window/card rectangles, and floating blobs
151    because the references are terminal-style screenshots.  The guide already has
152    the desired character/prop sketch, so generated strokes are treated as local
153    embellishment and are clipped back to the guide before the guide is merged.
154    """
155    try:
156        import cv2
157        import numpy as np
158
159        raw_mask = np.array(line_mask_from_terminal_art(art), dtype=np.uint8)
160        guide = np.array(guide_mask, dtype=np.uint8)
161        kernel_size = max(1, padding * 2 + 1)
162        kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (kernel_size, kernel_size))
163        keep_zone = cv2.dilate(guide, kernel, iterations=1)
164        clipped = cv2.bitwise_and(raw_mask, keep_zone)
165
166        cleaned = np.zeros_like(clipped)
167        label_count, labels, stats, _ = cv2.connectedComponentsWithStats(clipped, connectivity=8)
168        for label in range(1, label_count):
169            x, y, w, h, area = stats[label]
170            if area < 8:
171                continue
172            component = labels == label
173            density = area / max(1, w * h)
174
175            # Filled SDXL blobs make the final panels look less like the clean
176            # reference sketches. The exact guide is merged later, so dense raw
177            # fills can be rejected even when they overlap a desired guide mark.
178            if density > 0.34 and area > 24:
179                continue
180            if w > raw_mask.shape[1] * 0.38 and h < raw_mask.shape[0] * 0.04:
181                continue
182            cleaned[component] = 255
183
184        return mask_to_terminal_art(Image.fromarray(cleaned, "L"), art.size)
185    except Exception:
186        clipped = ImageChops.multiply(line_mask_from_terminal_art(art), guide_mask.filter(ImageFilter.MaxFilter(padding | 1)))
187        return mask_to_terminal_art(clipped, art.size)
188
189
190def merge_control_guide(art: Image.Image, raw_art_path: Path, panel_number: int) -> Image.Image:
191    control_path = raw_art_path.with_name(f"control-panel-{panel_number:02d}.png")
192    guide = control_guide_line_art(control_path, art.size)
193    guide_mask = control_guide_mask(control_path, art.size)
194    if guide is None or guide_mask is None:
195        return art
196    # The ControlNet guide is derived from art-reference sprites. Keeping it in
197    # the final line art repairs SDXL drift. Generated strokes are now kept only
198    # to that guide, which removes scanlines/floating artifacts while retaining
199    # the cleaned SDXL panel generation as the raw/debug artifact.
200    filtered = filter_line_art_to_control_guide(art.convert("RGB"), guide_mask)
201    return ImageChops.lighter(filtered, guide)
202
203
204def wrap_for_width(draw: ImageDraw.ImageDraw, text: str, font: ImageFont.ImageFont, max_width: int) -> list[str]:
205    text = clean_markup(text)
206    words = text.split()
207    lines: list[str] = []
208    current: list[str] = []
209    for word in words:
210        candidate = " ".join([*current, word])
211        if draw.textlength(candidate, font=font) <= max_width or not current:
212            current.append(word)
213        else:
214            lines.append(" ".join(current))
215            current = [word]
216    if current:
217        lines.append(" ".join(current))
218    return lines or [""]
219
220
221def draw_tail(draw: ImageDraw.ImageDraw, box: tuple[int, int, int, int], speaker: str, outline: str):
222    x0, y0, x1, y1 = box
223    if speaker == "dev":
224        points = [(x0 + 80, y1 - 2), (x0 + 105, y1 + 34), (x0 + 145, y1 - 2)]
225    elif speaker == "clanker":
226        points = [(x1 - 145, y1 - 2), (x1 - 105, y1 + 34), (x1 - 80, y1 - 2)]
227    else:
228        return
229    draw.polygon(points, fill=BLACK_ALPHA, outline=outline)
230
231
232def draw_bubble(
233    overlay: Image.Image,
234    text: str,
235    speaker: str,
236    box: tuple[int, int, int, int],
237    font: ImageFont.ImageFont,
238):
239    draw = ImageDraw.Draw(overlay)
240    x0, y0, x1, y1 = box
241    draw.rounded_rectangle(box, radius=18, fill=BLACK_ALPHA, outline=GREEN, width=4)
242    draw_tail(draw, box, speaker, GREEN)
243
244    label = "" if speaker == "narrator" else f"{speaker.upper()}: "
245    lines = wrap_for_width(draw, f"{label}{text}", font, (x1 - x0) - 36)
246    line_height = int(font.size * 0.9) if hasattr(font, "size") else 24
247    total_height = len(lines) * line_height
248    y = y0 + max(12, ((y1 - y0) - total_height) // 2)
249    for line in lines:
250        draw.text((x0 + 18, y), line, font=font, fill=LIGHT_GREEN)
251        y += line_height
252
253
254def bubble_box(index: int, total: int, speaker: str, image_width: int) -> tuple[int, int, int, int]:
255    top = 28 + index * 116
256    height = 92
257    margin = 34
258    if speaker == "dev":
259        return (margin, top, int(image_width * 0.62), top + height)
260    if speaker == "clanker":
261        return (int(image_width * 0.38), top, image_width - margin, top + height)
262    return (margin, top, image_width - margin, top + height)
263
264
265def draw_terminal(
266    overlay: Image.Image,
267    lines: list[str],
268    project_root: Path,
269):
270    if not lines:
271        return
272    draw = ImageDraw.Draw(overlay)
273    width, height = overlay.size
274    font = load_font(project_root, 34)
275    clean_lines = [clean_markup(line) for line in lines[:4]]
276    line_height = 35
277    x = 56
278    y = height - (len(clean_lines) * line_height) - 48
279    # Render only the joke's code/output text. Avoid terminal-window chrome,
280    # title bars, boxed footers, or UI panels inside the comic image.
281    for line in clean_lines:
282        shortened = textwrap.shorten(line, width=42, placeholder="…")
283        draw.text(
284            (x, y),
285            shortened,
286            font=font,
287            fill=LIGHT_GREEN,
288            stroke_width=2,
289            stroke_fill=(0, 0, 0, 230),
290        )
291        y += line_height
292
293
294def composite_panel(
295    raw_art_path: Path,
296    output_path: Path,
297    panel: PanelSpec,
298    project_root: Path,
299    panel_number: int,
300    art_postprocess_mode: str = "terminal-lineart",
301    line_art_threshold: int = 95,
302    line_art_stroke: int = 3,
303) -> Path:
304    raw = Image.open(raw_art_path).convert("RGB")
305    base_art = prepare_art(
306        raw,
307        mode=art_postprocess_mode,
308        threshold=line_art_threshold,
309        stroke=line_art_stroke,
310    )
311    if art_postprocess_mode == "terminal-lineart":
312        base_art = merge_control_guide(base_art, raw_art_path, panel_number)
313    base = base_art.convert("RGBA")
314    overlay = Image.new("RGBA", base.size, (0, 0, 0, 0))
315    draw = ImageDraw.Draw(overlay)
316    width, height = base.size
317
318    # Simple comic panel frame. No baked-in panel number, title card, header,
319    # footer, or site chrome; the page provides that in HTML.
320    draw.rectangle((8, 8, width - 8, height - 8), outline=GREEN, width=3)
321
322    bubble_font = load_font(project_root, 42)
323    items: list[DialogueLine | tuple[str, str]] = []
324    if panel.caption:
325        items.append(("narrator", panel.caption))
326    items.extend(panel.dialogue)
327
328    for index, item in enumerate(items[:3]):
329        if isinstance(item, tuple):
330            speaker, text = item
331        else:
332            speaker, text = item.speaker, item.text
333        draw_bubble(overlay, text, speaker, bubble_box(index, len(items), speaker, width), bubble_font)
334
335    draw_terminal(overlay, panel.terminalLines, project_root)
336
337    composed = Image.alpha_composite(base, overlay).convert("RGB")
338    output_path.parent.mkdir(parents=True, exist_ok=True)
339    composed.save(output_path, quality=95)
340    return output_path