@@ -3,6 +3,9 @@
33
44 This script intentionally generates only raw image panels. It does not compose speech
55 bubbles, captions, terminal text, borders, or final strips.
6+
7+The optional SDXL ControlNet path uses provided or existing control images as
8+conditioning. It does not draw final comic art itself.
69 """
710
811 from __future__ import annotations
@@ -16,14 +19,20 @@ from datetime import datetime, timezone
1619 from pathlib import Path
1720 from typing import Any
1821
22+from PIL import Image
23+
1924 DEFAULT_MODEL = "stabilityai/stable-diffusion-xl-base-1.0"
2025 DEFAULT_VAE = "madebyollin/sdxl-vae-fp16-fix"
21-DEFAULT_STYLE_PROMPT = "ctdstyle, neon green outline doodle, pure black background, simple sparse comic"
26+DEFAULT_CONTROLNET_MODEL = "diffusers/controlnet-canny-sdxl-1.0"
27+DEFAULT_STYLE_PROMPT = (
28+ "ctdstyle, simple neon green monoline stick figure comic, pure black background, "
29+ "thin glowing outlines, sparse raw panel art matching the art-reference sheets"
30+)
2231 DEFAULT_CHARACTER_PROMPT = (
23- "Dev: stick figure developer, round head, rectangular glasses, thin limbs. "
24- "Clanker: small boxy robot, screen head, antenna, claw hands"
32+ "Dev is a minimalist stick figure software developer with a round head, rectangular glasses, and thin limbs. "
33+ "Clanker is a small cute boxy robot assistant with a rectangular screen head, antenna, segmented arms, and claw hands."
2534 )
26-DEFAULT_PANEL_SUFFIX = "raw art only, no text, no speech bubbles"
35+DEFAULT_PANEL_SUFFIX = "raw art only, no text, no speech bubbles, no captions, no panel borders"
2736 DEFAULT_NEGATIVE_PROMPT = (
2837 "anime, manga, cyberpunk, armored suit, helmet, realistic human, hoodie, detailed clothing, "
2938 "text, words, letters, speech bubble, caption, watermark, logo, panel border, white background, "
@@ -35,11 +44,26 @@ DEFAULT_NEGATIVE_PROMPT = (
3544 @dataclass(frozen=True)
3645 class PanelJob:
3746 index: int
47+ variant_index: int
48+ ordinal: int
3849 art_prompt: str
3950 output_path: Path
4051 seed: int
4152
4253
54+@dataclass(frozen=True)
55+class ControlImagePlan:
56+ image: Image.Image
57+ source_path: Path
58+ saved_path: Path | None
59+
60+
61+@dataclass(frozen=True)
62+class InitImagePlan:
63+ image: Image.Image
64+ source: str
65+
66+
4367 def project_paths() -> tuple[Path, Path]:
4468 comic_root = Path(__file__).resolve().parent
4569 site_root = comic_root.parent
@@ -55,6 +79,22 @@ def display_path(path: Path) -> str:
5579 return str(path)
5680
5781
82+def resolve_input_path(path: Path, spec_path: Path | None = None) -> Path:
83+ if path.is_absolute():
84+ return path
85+
86+ comic_root, site_root = project_paths()
87+ candidates = [Path.cwd() / path]
88+ if spec_path is not None:
89+ candidates.append(spec_path.parent / path)
90+ candidates.extend([comic_root / path, site_root / path])
91+
92+ for candidate in candidates:
93+ if candidate.exists():
94+ return candidate
95+ return candidates[0]
96+
97+
5898 def parse_args() -> argparse.Namespace:
5999 comic_root, site_root = project_paths()
60100
@@ -69,12 +109,13 @@ def parse_args() -> argparse.Namespace:
69109
70110 parser.add_argument("--pretrained-model-name-or-path", default=DEFAULT_MODEL)
71111 parser.add_argument("--vae-model-name-or-path", default=DEFAULT_VAE)
72- parser.add_argument("--lora-path", type=Path, default=comic_root / "models" / "ctdstyle-sdxl-lora")
112+ parser.add_argument("--lora-path", type=Path, default=comic_root / "models" / "ctdstyle-sdxl-lora-v3")
73113 parser.add_argument("--no-lora", action="store_true")
74114 parser.add_argument("--lora-scale", type=float, default=0.85)
75115 parser.add_argument("--hf-home", type=Path, default=comic_root / ".cache" / "huggingface")
76116
77117 parser.add_argument("--style-prompt", default=DEFAULT_STYLE_PROMPT)
118+ parser.add_argument("--style-reference-dir", type=Path, default=site_root / "art-reference")
78119 parser.add_argument("--character-prompt", default=DEFAULT_CHARACTER_PROMPT)
79120 parser.add_argument("--character-prompt-file", type=Path, help="Optional text file overriding --character-prompt")
80121 parser.add_argument("--no-character-prompt", action="store_true", help="Do not inject shared Dev/Clanker descriptions")
@@ -106,7 +147,80 @@ def parse_args() -> argparse.Namespace:
106147 parser.add_argument("--print-prompts", action="store_true", help="Print full prompts as panels are planned")
107148 parser.add_argument("--cpu-offload", action=argparse.BooleanOptionalAction, default=True)
108149 parser.add_argument("--mixed-precision", choices=["fp16", "bf16", "no"], default="fp16")
109- return parser.parse_args()
150+ controlnet_group = parser.add_argument_group("ControlNet")
151+ controlnet_group.add_argument("--controlnet", action="store_true", help="Use StableDiffusionXLControlNetPipeline")
152+ controlnet_group.add_argument("--controlnet-model-or-path", default=DEFAULT_CONTROLNET_MODEL)
153+ controlnet_group.add_argument("--controlnet-scale", type=float, default=0.8)
154+ controlnet_group.add_argument("--controlnet-start", type=float, default=0.0)
155+ controlnet_group.add_argument("--controlnet-end", type=float, default=0.9)
156+ controlnet_group.add_argument(
157+ "--control-mode",
158+ choices=["existing", "image"],
159+ default="existing",
160+ help="existing uses panel.controlImage or assets/comics/<slug>/control-panel-XX.png; image requires --control-image.",
161+ )
162+ controlnet_group.add_argument(
163+ "--control-image",
164+ type=Path,
165+ action="append",
166+ default=[],
167+ help="Control image path. Use once for all panels or once per selected generated image. Specs may also set panel.controlImage.",
168+ )
169+ controlnet_group.add_argument(
170+ "--control-preprocess",
171+ choices=["lineart-threshold", "none"],
172+ default="lineart-threshold",
173+ help="How to convert provided control images before sending them to ControlNet.",
174+ )
175+ controlnet_group.add_argument("--control-threshold", type=int, default=24, help="Brightness threshold for lineart-threshold preprocessing")
176+ controlnet_group.add_argument(
177+ "--control-resize-mode",
178+ choices=["fit", "fill", "stretch"],
179+ default="fit",
180+ help="Resize behavior for provided control images.",
181+ )
182+ controlnet_group.add_argument(
183+ "--save-control-images",
184+ action=argparse.BooleanOptionalAction,
185+ default=False,
186+ help="Save preprocessed ControlNet inputs next to generated raw panels as controlnet-input-panel-XX.png.",
187+ )
188+ controlnet_group.add_argument(
189+ "--controlnet-init-mode",
190+ choices=["none", "black", "image"],
191+ default="none",
192+ help="Use ControlNet img2img instead of txt2img. black starts from a plain black canvas; image uses --init-image.",
193+ )
194+ controlnet_group.add_argument("--init-image", type=Path, help="Initial image for --controlnet-init-mode=image")
195+ controlnet_group.add_argument("--strength", type=float, default=0.98, help="Denoising strength for ControlNet img2img")
196+
197+ args = parser.parse_args()
198+ validate_args(args)
199+ return args
200+
201+
202+def validate_args(args: argparse.Namespace) -> None:
203+ if args.width <= 0 or args.height <= 0:
204+ raise SystemExit("--width and --height must be positive")
205+ if args.num_images_per_panel <= 0:
206+ raise SystemExit("--num-images-per-panel must be positive")
207+ if args.controlnet:
208+ if not 0.0 <= args.controlnet_start <= 1.0:
209+ raise SystemExit("--controlnet-start must be between 0 and 1")
210+ if not 0.0 <= args.controlnet_end <= 1.0:
211+ raise SystemExit("--controlnet-end must be between 0 and 1")
212+ if args.controlnet_end < args.controlnet_start:
213+ raise SystemExit("--controlnet-end must be >= --controlnet-start")
214+ if args.controlnet_scale < 0:
215+ raise SystemExit("--controlnet-scale must be non-negative")
216+ if not 0 <= args.control_threshold <= 255:
217+ raise SystemExit("--control-threshold must be between 0 and 255")
218+ if args.controlnet_init_mode != "none" and not 0.0 < args.strength <= 1.0:
219+ raise SystemExit("--strength must be > 0 and <= 1 for ControlNet img2img")
220+ if args.controlnet_init_mode == "image" and args.init_image is None:
221+ raise SystemExit("--controlnet-init-mode=image requires --init-image")
222+ elif args.controlnet_init_mode != "none":
223+ raise SystemExit("--controlnet-init-mode requires --controlnet")
110224
111225
112226 def resolve_specs(args: argparse.Namespace) -> list[Path]:
@@ -141,6 +255,13 @@ def resolve_character_prompt(args: argparse.Namespace) -> str:
141255 return args.character_prompt.strip()
142256
143257
258+def style_reference_paths(args: argparse.Namespace) -> list[Path]:
259+ style_reference_dir = resolve_input_path(args.style_reference_dir)
260+ if not style_reference_dir.exists():
261+ return []
262+ return sorted(path for path in style_reference_dir.iterdir() if path.suffix.lower() in {".png", ".jpg", ".jpeg", ".webp"})
263+
264+
144265 def join_prompt_parts(*parts: str) -> str:
145266 return ", ".join(part.strip() for part in parts if part and part.strip())
146267
@@ -173,6 +294,16 @@ def output_name(panel_index: int, variant_index: int, num_variants: int) -> str:
173294 return f"raw-panel-{panel_index:02d}-{variant_index:02d}.png"
174295
175296
297+def control_output_name(panel_index: int, variant_index: int, num_variants: int) -> str:
298+ if num_variants == 1:
299+ return f"controlnet-input-panel-{panel_index:02d}.png"
300+ return f"controlnet-input-panel-{panel_index:02d}-{variant_index:02d}.png"
301+
302+
303+def existing_control_name(panel_index: int) -> str:
304+ return f"control-panel-{panel_index:02d}.png"
305+
306+
176307 def plan_jobs(spec: dict[str, Any], spec_output_dir: Path, args: argparse.Namespace) -> list[PanelJob]:
177308 requested_panels = set(args.panel or [])
178309 jobs: list[PanelJob] = []
@@ -185,6 +316,8 @@ def plan_jobs(spec: dict[str, Any], spec_output_dir: Path, args: argparse.Namesp
185316 jobs.append(
186317 PanelJob(
187318 index=panel_index,
319+ variant_index=variant_index,
320+ ordinal=len(jobs),
188321 art_prompt=panel["artPrompt"],
189322 output_path=spec_output_dir / output_name(panel_index, variant_index, args.num_images_per_panel),
190323 seed=seed,
@@ -196,38 +329,89 @@ def plan_jobs(spec: dict[str, Any], spec_output_dir: Path, args: argparse.Namesp
196329 return jobs
197330
198331
332+def load_component_with_optional_variant(component: Any, model_name_or_path: str | Path, kwargs: dict[str, Any], variant: str | None) -> Any:
333+ load_kwargs = dict(kwargs)
334+ if variant:
335+ load_kwargs["variant"] = variant
336+ try:
337+ return component.from_pretrained(model_name_or_path, **load_kwargs)
338+ except (OSError, ValueError) as exc:
339+ if not variant:
340+ raise
341+ print(f"Could not load {model_name_or_path} with variant={variant!r}; retrying without variant ({exc})")
342+ load_kwargs.pop("variant", None)
343+ return component.from_pretrained(model_name_or_path, **load_kwargs)
344+
345+
346+def load_lora(pipe: Any, args: argparse.Namespace) -> None:
347+ if args.no_lora:
348+ return
349+ if not args.lora_path.exists():
350+ raise FileNotFoundError(f"LoRA path not found: {args.lora_path}")
351+ if args.lora_path.is_file():
352+ pipe.load_lora_weights(str(args.lora_path.parent), weight_name=args.lora_path.name, adapter_name="ctdstyle")
353+ else:
354+ lora_kwargs: dict[str, Any] = {}
355+ default_weight = args.lora_path / "pytorch_lora_weights.safetensors"
356+ if default_weight.exists():
357+ lora_kwargs["weight_name"] = default_weight.name
358+ pipe.load_lora_weights(str(args.lora_path), adapter_name="ctdstyle", **lora_kwargs)
359+ if hasattr(pipe, "set_adapters"):
360+ pipe.set_adapters(["ctdstyle"], adapter_weights=[args.lora_scale])
361+
362+
199363 def load_pipeline(args: argparse.Namespace) -> Any:
200364 import torch
201- from diffusers import AutoencoderKL, DPMSolverMultistepScheduler, StableDiffusionXLPipeline
365+ from diffusers import (
366+ AutoencoderKL,
367+ ControlNetModel,
368+ DPMSolverMultistepScheduler,
369+ StableDiffusionXLControlNetImg2ImgPipeline,
370+ StableDiffusionXLControlNetPipeline,
371+ StableDiffusionXLPipeline,
372+ )
202373
203374 dtype = {"fp16": torch.float16, "bf16": torch.bfloat16, "no": torch.float32}[args.mixed_precision]
375+ variant = "fp16" if args.mixed_precision == "fp16" else None
204376
205377 vae = None
206378 if args.vae_model_name_or_path:
207379 vae = AutoencoderKL.from_pretrained(args.vae_model_name_or_path, torch_dtype=dtype)
208380
209- pipe = StableDiffusionXLPipeline.from_pretrained(
210- args.pretrained_model_name_or_path,
211- vae=vae,
212- torch_dtype=dtype,
213- variant="fp16" if args.mixed_precision == "fp16" else None,
214- use_safetensors=True,
215- )
216- pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
381+ common_kwargs: dict[str, Any] = {
382+ "vae": vae,
383+ "torch_dtype": dtype,
384+ "use_safetensors": True,
385+ }
386+
387+ if args.controlnet:
388+ controlnet = load_component_with_optional_variant(
389+ ControlNetModel,
390+ args.controlnet_model_or_path,
391+ {"torch_dtype": dtype, "use_safetensors": True},
392+ variant,
393+ )
394+ pipeline_class = (
395+ StableDiffusionXLControlNetImg2ImgPipeline
396+ if args.controlnet_init_mode != "none"
397+ else StableDiffusionXLControlNetPipeline
398+ )
399+ pipe = load_component_with_optional_variant(
400+ pipeline_class,
401+ args.pretrained_model_name_or_path,
402+ {**common_kwargs, "controlnet": controlnet},
403+ variant,
404+ )
405+ else:
406+ pipe = load_component_with_optional_variant(
407+ StableDiffusionXLPipeline,
408+ args.pretrained_model_name_or_path,
409+ common_kwargs,
410+ variant,
411+ )
217412
218- if not args.no_lora:
219- if not args.lora_path.exists():
220- raise FileNotFoundError(f"LoRA path not found: {args.lora_path}")
221- if args.lora_path.is_file():
222- pipe.load_lora_weights(str(args.lora_path.parent), weight_name=args.lora_path.name, adapter_name="ctdstyle")
223- else:
224- lora_kwargs: dict[str, Any] = {}
225- default_weight = args.lora_path / "pytorch_lora_weights.safetensors"
226- if default_weight.exists():
227- lora_kwargs["weight_name"] = default_weight.name
228- pipe.load_lora_weights(str(args.lora_path), adapter_name="ctdstyle", **lora_kwargs)
229- if hasattr(pipe, "set_adapters"):
230- pipe.set_adapters(["ctdstyle"], adapter_weights=[args.lora_scale])
413+ pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
414+ load_lora(pipe, args)
231415
232416 if args.cpu_offload:
233417 pipe.enable_model_cpu_offload()
@@ -237,16 +421,158 @@ def load_pipeline(args: argparse.Namespace) -> Any:
237421 return pipe
238422
239423
240-def remove_stale_raw_panels(spec_output_dir: Path, jobs: list[PanelJob], args: argparse.Namespace) -> None:
424+def composite_on_black(image: Image.Image) -> Image.Image:
425+ if image.mode == "RGBA":
426+ background = Image.new("RGBA", image.size, (0, 0, 0, 255))
427+ return Image.alpha_composite(background, image).convert("RGB")
428+ return image.convert("RGB")
429+
430+
431+def resize_control_image(image: Image.Image, width: int, height: int, mode: str) -> Image.Image:
432+ image = composite_on_black(image)
433+ if image.size == (width, height):
434+ return image
435+
436+ if mode == "stretch":
437+ return image.resize((width, height), Image.Resampling.LANCZOS)
438+
439+ source_width, source_height = image.size
440+ if mode == "fill":
441+ scale = max(width / source_width, height / source_height)
442+ resized = image.resize((round(source_width * scale), round(source_height * scale)), Image.Resampling.LANCZOS)
443+ left = (resized.width - width) // 2
444+ top = (resized.height - height) // 2
445+ return resized.crop((left, top, left + width, top + height))
446+
447+ scale = min(width / source_width, height / source_height)
448+ resized = image.resize((round(source_width * scale), round(source_height * scale)), Image.Resampling.LANCZOS)
449+ canvas = Image.new("RGB", (width, height), "black")
450+ canvas.paste(resized, ((width - resized.width) // 2, (height - resized.height) // 2))
451+ return canvas
452+
453+
454+def lineart_threshold(image: Image.Image, threshold: int) -> Image.Image:
455+ mask = image.convert("L").point(lambda value: 255 if value > threshold else 0)
456+ return Image.merge("RGB", (mask, mask, mask))
457+
458+
459+def preprocess_control_image(image: Image.Image, args: argparse.Namespace) -> Image.Image:
460+ image = resize_control_image(image, args.width, args.height, args.control_resize_mode)
461+ if args.control_preprocess == "lineart-threshold":
462+ return lineart_threshold(image, args.control_threshold)
463+ return image
464+
465+
466+def load_control_image(path: Path, args: argparse.Namespace) -> Image.Image:
467+ if not path.exists():
468+ raise FileNotFoundError(f"Control image not found: {path}")
469+ return preprocess_control_image(Image.open(path), args)
470+
471+
472+def control_image_arg_for_job(args: argparse.Namespace, job: PanelJob, job_count: int) -> Path | None:
473+ if not args.control_image:
474+ return None
475+ if len(args.control_image) == 1:
476+ return args.control_image[0]
477+ if len(args.control_image) == job_count:
478+ return args.control_image[job.ordinal]
479+ raise ValueError(
480+ f"Expected either one --control-image or {job_count} --control-image values for selected jobs; got {len(args.control_image)}"
481+ )
482+
483+
484+def resolve_control_path(
485+ spec_path: Path,
486+ spec: dict[str, Any],
487+ spec_output_dir: Path,
488+ job: PanelJob,
489+ job_count: int,
490+ args: argparse.Namespace,
491+) -> Path:
492+ panel = spec["panels"][job.index - 1]
493+ control_arg = control_image_arg_for_job(args, job, job_count)
494+ panel_control = panel.get("controlImage") or panel.get("control_image")
495+
496+ if control_arg is not None:
497+ return resolve_input_path(control_arg, spec_path)
498+ if panel_control:
499+ return resolve_input_path(Path(str(panel_control)), spec_path)
500+ if args.control_mode == "existing":
501+ return spec_output_dir / existing_control_name(job.index)
502+ raise ValueError(f"Panel {job.index} needs --control-image or panel.controlImage when --control-mode=image")
503+
504+
505+def prepare_control_image(
506+ spec_path: Path,
507+ spec: dict[str, Any],
508+ spec_output_dir: Path,
509+ job: PanelJob,
510+ job_count: int,
511+ args: argparse.Namespace,
512+) -> ControlImagePlan | None:
513+ if not args.controlnet:
514+ return None
515+
516+ source_path = resolve_control_path(spec_path, spec, spec_output_dir, job, job_count, args)
517+ image = load_control_image(source_path, args)
518+ saved_path = None
519+ if args.save_control_images:
520+ saved_path = spec_output_dir / control_output_name(job.index, job.variant_index, args.num_images_per_panel)
521+ return ControlImagePlan(image=image, source_path=source_path, saved_path=saved_path)
522+
523+
524+def prepare_init_image(spec_path: Path, args: argparse.Namespace) -> InitImagePlan | None:
525+ if args.controlnet_init_mode == "none":
526+ return None
527+ if args.controlnet_init_mode == "black":
528+ return InitImagePlan(image=Image.new("RGB", (args.width, args.height), "black"), source="black-canvas")
529+ if args.init_image is None:
530+ raise RuntimeError("--controlnet-init-mode=image requires --init-image")
531+ source_path = resolve_input_path(args.init_image, spec_path)
532+ if not source_path.exists():
533+ raise FileNotFoundError(f"Init image not found: {source_path}")
534+ image = resize_control_image(Image.open(source_path), args.width, args.height, args.control_resize_mode)
535+ return InitImagePlan(image=image, source=display_path(source_path))
536+
537+
538+def remove_stale_outputs(spec_output_dir: Path, jobs: list[PanelJob], args: argparse.Namespace) -> None:
241539 if args.dry_run or not args.overwrite or not args.clean_stale or args.panel:
242540 return
243541
244- expected = {job.output_path.resolve() for job in jobs}
542+ expected_raw = {job.output_path.resolve() for job in jobs}
245543 for path in sorted(spec_output_dir.glob("raw-panel-*.png")):
246- if path.resolve() not in expected:
544+ if path.resolve() not in expected_raw:
247545 print(f"remove stale {path}")
248546 path.unlink()
249547
548+ if args.controlnet and args.save_control_images:
549+ expected_control = {
550+ (spec_output_dir / control_output_name(job.index, job.variant_index, args.num_images_per_panel)).resolve()
551+ for job in jobs
552+ }
Cut short: this commit's diff is over 1000 lines.