Merge pull request #239 from zero01101/controlnet_inpaint

controlnet inpaint

README.md

@@ -23,6 +23,7 @@ this is a completely vanilla javascript and html canvas outpainting convenience
 - arbitrary dream reticle size - draw the rectangle of your dreams
 - an [effectively infinite](https://github.com/zero01101/openOutpaint/pull/108), resizable, scalable canvas for you to paint all over
   - **_NOTE: v0.0.10 introduces a new "camera control" modifier key - hold [`CTRL`] and use the scrollwheel to zoom (scroll the wheel or use the two-finger vertical gesture on, uh, modern touchpads) and pan (hold the scrollwheel button, or if you don't have one, left-click button) around the canvas_**
+- extremely limited, janky support for a shockingly restrictive list of A1111 extensions including controlnet inpainting for legitimately [magic](https://github.com/Mikubill/sd-webui-controlnet/discussions/1464) [promptless inpainting](https://github.com/Mikubill/sd-webui-controlnet/discussions/1143) and [outpainting](https://github.com/Mikubill/sd-webui-controlnet/discussions/1597)
 - a very nicely functional and familiar layer system
 - save, load, import, and export workspaces - includes all your layers, history, canvas size, you name it!
 - inpainting/touchup mask brush

css/icons.css

@@ -128,6 +128,12 @@
 	mask-image: url("../res/icons/slice.svg");
 }
 
+.ui.inline-icon.icon-joystick::after,
+.ui.icon > .icon-joystick {
+	-webkit-mask-image: url("../res/icons/joystick.svg");
+	mask-image: url("../res/icons/joystick.svg");
+}
+
 .ui.inline-icon.icon-save::after,
 .ui.icon > .icon-save {
 	-webkit-mask-image: url("../res/icons/save.svg");

defaultscripts.json

@@ -9,8 +9,8 @@
 			"scriptValues": "[false, false, \"positive\", \"comma\", 2]"
 		},
 		"X/Y/Z plot": {
-			"titleText": "Params:\nx_type (int): index of axis type (see below) //def: 3\nx_values (mixed, str) //def: \"0.00-0.99 [4]\"\ny_type (int) //def: 4\ny_values (mixed, str) //def: \"5-30 [4]\"\nz_type (int) //def: 6\nz_values (mixed, str) //def: \"2.5-12.5 [4]\"\ndraw_legend (bool): return grid of all images //def: false\ninclude_lone_images (bool): return individual images //def: true\ninclude_subgrids (bool) //def: false\nno_fixed_seeds (bool): use different seeds for each picture //def: false\nmargin_size (int): grid margins in px //def: 2\n\nAvailable axis types:\n0: Nothing\n1: Seed\n2: Var. seed\n3: Var. strength\n4: Steps\n5: Hires steps (txt2img only)\n6: CFG Scale\n7: Image CFG Scale (img2img with instructPix2Pix only)\n8: Prompt S/R\n9: Prompt order\n10: Sampler (txt2img only)\n11: Sampler (img2img only)\n12: Checkpoint Name\n13: Sigma Churn\n14: Sigma min\n15: Sigma max\n16: Sigma noise\n17: Eta\n18: Clip skip\n19: Denoising\n20: Hires upscaler (txt2img only)\n21: Cond. Image Mask Weight (img2img only)\n22: VAE\n23: Styles\n24: UniPC Order\n25: Face Restore",
-			"scriptValues": "[3, \"0.00-0.99 [4]\", 4, \"5-30 [4]\", 6, \"2.5-12.5 [4]\", false, true, false, false, 2]"
+			"titleText": "Params:\nx_type (int): index of axis type (see below) //def: 3\nx_values (mixed, str) //def: \"0.00-0.99 [4]\"\nx_values_dropdown (NOIDEA) //def: \"\"\ny_type (int) //def: 4\ny_values (mixed, str) //def: \"5-30 [4]\"\ny_values_dropdown (NOIDEA) //def: \"\"\nz_type (int) //def: 6\nz_values (mixed, str) //def: \"2.5-12.5 [4]\"\nz_values_dropdown (NOIDEA) //def: \"\"\ndraw_legend (bool): return grid of all images //def: false\ninclude_lone_images (bool): return individual images //def: true\ninclude_subgrids (bool) //def: false\nno_fixed_seeds (bool): use different seeds for each picture //def: false\nmargin_size (int): grid margins in px //def: 2\n\nAvailable axis types:\n0: Nothing\n1: Seed\n2: Var. seed\n3: Var. strength\n4: Steps\n5: Hires steps (txt2img only)\n6: CFG Scale\n7: Image CFG Scale (img2img with instructPix2Pix only)\n8: Prompt S/R\n9: Prompt order\n10: Sampler (txt2img only)\n11: Sampler (img2img only)\n12: Checkpoint Name\n13: Negative Guidance minimum sigma\n14: Sigma Churn\n15: Sigma min\n16: Sigma max\n17: Sigma noise\n18: Schedule Type\n19: Schedule min sigma\n20:Schedule max sigma\n21: Schedule rho\n22: Eta\n23: Clip skip\n24: Denoising\n25: Hires upscaler (txt2img only)\n26: Cond. Image Mask Weight (img2img only)\n27: VAE\n23: Styles\n28: UniPC Order\n29: Face Restore\n30: Token merging ratio\n31: Token merging ratio hi-res",
+			"scriptValues": "[3, \"0.00-0.99 [4]\", \"\", 4, \"5-30 [4]\", \"\", 6, \"2.5-12.5 [4]\", \"\", false, true, false, false, 2]"
 		}
 	}
 }

index.html

@@ -5,7 +5,7 @@
 		<title>openOutpaint 🐠</title>
 		<!-- CSS Variables -->
 		<link href="css/colors.css?v=f732f19" rel="stylesheet" />
-		<link href="css/icons.css?v=a8ec439" rel="stylesheet" />
+		<link href="css/icons.css?v=e6f94af" rel="stylesheet" />
 
 		<link href="css/index.css?v=aaad3e5" rel="stylesheet" />
 		<link href="css/layers.css?v=92c0352" rel="stylesheet" />
@@ -143,10 +143,12 @@
 						id="seed"
 						onchange="changeSeed()"
 						min="-1"
-						max="9999999999"
+						max="99999999999999999999"
 						value="-1"
 						step="1" />
 					<br />
+					<label>Lora:</label>
+					<div id="lora-ac-select"></div>
 					<input type="checkbox" id="cbxHRFix" onchange="changeHiResFix()" />
 					<label for="cbxHRFix">Apply Txt2Img HRfix</label>
 					<br />
@@ -195,7 +197,48 @@
 						step="1"
 						onchange="changeMaskBlur()" />
 				</div>
-
+				<!-- Extensions section -->
+				<button type="button" class="collapsible">Extensions</button>
+				<div class="content">
+					<input
+						type="checkbox"
+						id="cbxDynPrompts"
+						onchange="changeDynamicPromptsExtension()"
+						disabled="disabled" />
+					<label for="cbxDynPrompts">Dynamic Prompts</label>
+					<br />
+					<input
+						type="checkbox"
+						id="cbxControlNet"
+						onchange="changeControlNetExtension()"
+						disabled="disabled" />
+					<label for="cbxControlNet">ControlNet In/Outpainting</label>
+					<br class="controlnetElement" />
+					<label id="cnModuleLabel" class="controlnetElement">
+						Preprocessor
+					</label>
+					<div id="controlNetModule-ac-select" class="controlnetElement"></div>
+					<label id="cnModelLabel" class="controlnetElement">Model</label>
+					<div id="controlNetModel-ac-select" class="controlnetElement"></div>
+					<label id="cnControlLabel" class="controlnetElement">
+						Control Mode
+					</label>
+					<select id="controlNetMode-select" class="controlnetElement">
+						<option value="Balanced">balanced</option>
+						<option value="My prompt is more important">+prompt</option>
+						<option value="ControlNet is more important">+CN</option>
+					</select>
+					<br />
+					<label id="cnResizeLabel" class="controlnetElement">
+						Resize Mode
+					</label>
+					<select id="controlNetResize-select" class="controlnetElement">
+						<option value="Just Resize">resize</option>
+						<option value="Crop and Resize">+crop</option>
+						<option value="Resize and Fill">+fill</option>
+					</select>
+					<!-- <div id="referenceStyleFidelity" class="controlnetElement"></div> -->
+				</div>
 				<!-- Save/load image section -->
 				<button type="button" class="collapsible">Save/Upscaling</button>
 				<div class="content">
@@ -244,7 +287,8 @@
 					<br />
 					<span id="version">
 						<a href="https://github.com/zero01101/openOutpaint" target="_blank">
-							Alpha release v0.0.15.3 (20230410.001)
+							<s>Alpha release v0.0.16</s>
+							v20230708.001
 						</a>
 						<br />
 						<a
@@ -441,6 +485,7 @@
 		<!-- Basics -->
 		<script src="js/global.js?v=ac30d16" type="text/javascript"></script>
 		<script src="js/defaults.js?v=5b06818" type="text/javascript"></script>
+		<script src="js/extensions.js?v=fec7579" type="text/javascript"></script>
 
 		<!-- Base Libs -->
 		<script src="js/lib/util.js?v=379aef7" type="text/javascript"></script>
@@ -466,7 +511,7 @@
 
 		<!-- Content -->
 		<script src="js/prompt.js?v=7a1c68c" type="text/javascript"></script>
-		<script src="js/index.js?v=e10fcb1" type="text/javascript"></script>
+		<script src="js/index.js?v=31d52e7" type="text/javascript"></script>
 
 		<script
 			src="js/ui/floating/history.js?v=4f29db4"
@@ -480,7 +525,7 @@
 			src="js/ui/tool/generic.js?v=3e678e0"
 			type="text/javascript"></script>
 
-		<script src="js/ui/tool/dream.js?v=97e4806" type="text/javascript"></script>
+		<script src="js/ui/tool/dream.js?v=45b8acd" type="text/javascript"></script>
 		<script
 			src="js/ui/tool/maskbrush.js?v=e9bd0eb"
 			type="text/javascript"></script>

js/extensions.js

@@ -0,0 +1,160 @@
+/**
+ * Extensions helper thing or class or whatever
+ */
+
+const extensions = {
+	// alwaysOnScriptsData: {},
+	alwaysOnScripts: false,
+	controlNetEnabled: false,
+	controlNetActive: false,
+	selectedControlNetModel: null,
+	selectedControlNetModule: null,
+	dynamicPromptsEnabled: false,
+	dynamicPromptsActive: false,
+	dynamicPromptsAlwaysonScriptName: null, //GRUMBLE GRUMBLE
+	enabledExtensions: [],
+	controlNetModels: null,
+	controlNetModules: null,
+
+	async getExtensions(
+		controlNetModelAutoComplete,
+		controlNetModuleAutoComplete
+	) {
+		const allowedExtensions = [
+			"controlnet",
+			// "none",
+			// "adetailer", // no API, can't verify available models
+			"dynamic prompts", //seriously >:( why put version in the name, now i have to fuzzy match it - just simply enabled or not? no API but so so good
+			//"segment anything", // ... API lets me get model but not processor?!?!?!
+			//"self attention guidance", // no API but useful, just enabled button, scale and threshold sliders?
+		];
+		// check http://127.0.0.1:7860/sdapi/v1/scripts for extensions
+		// if any of the allowed extensions are found, add them to the list
+		var url = document.getElementById("host").value + "/sdapi/v1/scripts";
+		try {
+			const response = await fetch(url);
+			const data = await response.json();
+			// enable checkboxes for extensions based on existence in data
+			data.img2img
+				.filter((extension) => {
+					return allowedExtensions.some((allowedExtension) => {
+						return extension.toLowerCase().includes(allowedExtension);
+					});
+				})
+				.forEach((extension) => {
+					this.enabledExtensions.push(extension);
+				});
+		} catch (e) {
+			console.warn("[index] Failed to fetch extensions");
+			console.warn(e);
+		}
+		this.checkForDynamicPrompts();
+		this.checkForControlNet(
+			controlNetModelAutoComplete,
+			controlNetModuleAutoComplete
+		);
+		//checkForSAM(); //or inpaintAnything or something i dunno
+		//checkForADetailer(); //? this one seems iffy
+		//checkForSAG(); //??
+	},
+
+	async checkForDynamicPrompts() {
+		if (
+			this.enabledExtensions.filter((e) => e.includes("dynamic prompts"))
+				.length > 0
+		) {
+			// Dynamic Prompts found, enable checkbox
+			this.alwaysOnScripts = true;
+			this.dynamicPromptsAlwaysonScriptName =
+				this.enabledExtensions[
+					this.enabledExtensions.findIndex((e) => e.includes("dynamic prompts"))
+				];
+			// this.alwaysOnScriptsData[this.dynamicPromptsAlwaysonScriptName] = {};
+			this.dynamicPromptsEnabled = true;
+			document.getElementById("cbxDynPrompts").disabled = false;
+		}
+		// basically param 0 is true for on, false for off, that's it
+	},
+
+	async checkForControlNet(
+		controlNetModelAutoComplete,
+		controlNetModuleAutoComplete
+	) {
+		var url = document.getElementById("host").value + "/controlnet/version";
+
+		try {
+			const response = await fetch(url);
+			const data = await response.json();
+
+			if (
+				data.version > 0 &&
+				this.enabledExtensions.filter((e) => e.includes("controlnet")).length >
+					0
+			) {
+				// ControlNet found
+				this.alwaysOnScripts = true;
+				this.controlNetEnabled = true;
+				document.getElementById("cbxControlNet").disabled = false;
+				// ok cool so now we can get the models and modules
+				this.getModels(controlNetModelAutoComplete);
+				this.getModules(controlNetModuleAutoComplete);
+			}
+		} catch (e) {
+			// ??
+			global.controlnetAPI = false;
+		}
+	},
+	async getModels(controlNetModelAutoComplete) {
+		// only worry about inpaint models for now
+		var url = document.getElementById("host").value + "/controlnet/model_list";
+
+		try {
+			const response = await fetch(url);
+			const data = await response.json();
+
+			this.controlNetModels = data.model_list;
+		} catch (e) {
+			console.warn("[extensions] Failed to fetch controlnet models");
+			console.warn(e);
+		}
+
+		let opt = null;
+		opt = this.controlNetModels
+			.filter((m) => m.includes("inpaint"))
+			.map((option) => ({
+				name: option,
+				value: option,
+			}));
+
+		controlNetModelAutoComplete.options = opt;
+	},
+	async getModules(controlNetModuleAutoComplete) {
+		const allowedModules = ["reference", "inpaint"];
+		var url = document.getElementById("host").value + "/controlnet/module_list";
+
+		try {
+			const response = await fetch(url);
+			const data = await response.json();
+
+			this.controlNetModules = data;
+		} catch (e) {
+			console.warn("[extensions] Failed to fetch controlnet modules");
+			console.warn(e);
+		}
+
+		let opt = null;
+		opt = this.controlNetModules.module_list
+			.filter((m) => m.includes("inpaint")) // why is there just "inpaint" in the modules if it's not in the ui
+			.map((option) => ({
+				name: option,
+				value: option,
+			}));
+
+		opt.push({
+			name: "inpaint_global_harmonious",
+			value: "inpaint_global_harmonious", // WTF WHY IS THIS ONE NOT LISTED IN MODULES BUT DISTINCT IN THE API CALL?!?!?!??!??! it is slightly different from "inpaint" from what i can tell
+		});
+
+		controlNetModuleAutoComplete.options = opt;
+	},
+};

js/index.js

@@ -171,6 +171,7 @@ function startup() {
 	changeHiResSquare();
 	changeRestoreFaces();
 	changeSyncCursorSize();
+	changeControlNetExtension();
 	checkFocus();
 	refreshScripts();
 }
@@ -421,6 +422,13 @@ async function testHostConnection() {
 							getSamplers();
 							getUpscalers();
 							getModels();
+							extensions.getExtensions(
+								controlNetModelAutoComplete,
+								controlNetModuleAutoComplete
+							);
+							getLoras();
+							// getTIEmbeddings();
+							// getHypernets();
 							firstTimeOnline = false;
 						}
 						break;
@@ -644,6 +652,18 @@ modelAutoComplete.onchange.on(({value}) => {
 		).style.backgroundColor = "#fcc";
 });
 
+let loraAutoComplete = createAutoComplete(
+	"LoRa",
+	document.getElementById("lora-ac-select")
+);
+loraAutoComplete.onchange.on(({value}) => {
+	// add selected lora to the end of the prompt
+	let passVal = "  <lora:" + value + ":1>";
+	let promptInput = document.getElementById("prompt");
+	promptInput.value += passVal;
+	let promptThing = prompt;
+});
+
 const samplerAutoComplete = createAutoComplete(
 	"Sampler",
 	document.getElementById("sampler-ac-select")
@@ -659,6 +679,29 @@ const hrFixUpscalerAutoComplete = createAutoComplete(
 	document.getElementById("hrFixUpscaler")
 );
 
+let controlNetModelAutoComplete = createAutoComplete(
+	"ControlNet Model",
+	document.getElementById("controlNetModel-ac-select")
+);
+
+controlNetModelAutoComplete.onchange.on(({value}) => {
+	extensions.selectedControlNetModel = value;
+});
+
+let controlNetModuleAutoComplete = createAutoComplete(
+	"ControlNet Module",
+	document.getElementById("controlNetModule-ac-select")
+);
+
+controlNetModuleAutoComplete.onchange.on(({value}) => {
+	extensions.selectedControlNetModule = value;
+});
+
+// const extensionsAutoComplete = createAutoComplete(
+// 	"Extension",
+// 	document.getElementById("extension-ac-select")
+// );
+
 hrFixUpscalerAutoComplete.onchange.on(({value}) => {
 	stableDiffusionData.hr_upscaler = value;
 	localStorage.setItem(`openoutpaint/hr_upscaler`, value);
@@ -798,6 +841,25 @@ function changeHRFY() {
 		document.getElementById("hr_resize_y").value;
 }
 
+function changeDynamicPromptsExtension() {
+	extensions.dynamicPromptsActive =
+		document.getElementById("cbxDynPrompts").checked;
+}
+
+function changeControlNetExtension() {
+	extensions.controlNetActive =
+		document.getElementById("cbxControlNet").checked;
+	if (extensions.controlNetActive) {
+		document
+			.querySelectorAll(".controlnetElement")
+			.forEach((el) => el.classList.remove("invisible"));
+	} else {
+		document
+			.querySelectorAll(".controlnetElement")
+			.forEach((el) => el.classList.add("invisible"));
+	}
+}
+
 function changeHiResFix() {
 	stableDiffusionData.enable_hr = Boolean(
 		document.getElementById("cbxHRFix").checked
@@ -1130,6 +1192,24 @@ async function getModels(refresh = false) {
 	}
 }
 
+async function getLoras() {
+	var url = document.getElementById("host").value + "/sdapi/v1/loras";
+	let opt = null;
+
+	try {
+		const response = await fetch(url);
+		const data = await response.json();
+
+		loraAutoComplete.options = data.map((lora) => ({
+			name: lora.name,
+			value: lora.name,
+		}));
+	} catch (e) {
+		console.warn("[index] Failed to fetch loras");
+		console.warn(e);
+	}
+}
+
 async function getConfig() {
 	var url = document.getElementById("host").value + "/sdapi/v1/options";
 
@@ -1247,6 +1327,7 @@ async function getSamplers() {
 		console.warn(e);
 	}
 }
+
 async function upscaleAndDownload() {
 	// Future improvements: some upscalers take a while to upscale, so we should show a loading bar or something, also a slider for the upscale amount
 

js/ui/tool/dream.js

@@ -1001,6 +1001,7 @@ const _generate = async (endpoint, request, bb, options = {}) => {
 const dream_generate_callback = async (bb, resolution, state) => {
 	// Build request to the API
 	const request = {};
+	const canvasTransport = {}; //this is the worst idea but i hate myself so i'm doing it anyway
 	Object.assign(request, stableDiffusionData);
 
 	request.width = resolution.w;
@@ -1013,8 +1014,125 @@ const dream_generate_callback = async (bb, resolution, state) => {
 	// Get visible pixels
 	const visibleCanvas = uil.getVisible(bb);
 
-	// Use txt2img if canvas is blank
-	if (isCanvasBlank(0, 0, bb.w, bb.h, visibleCanvas)) {
+	if (extensions.alwaysOnScripts) {
+		buildAlwaysOnScripts(state);
+	}
+
+	// Use txt2img if canvas is blank or if controlnet is active because "Allow inpaint in txt2img. This is necessary because txt2img has high-res fix" as per https://github.com/Mikubill/sd-webui-controlnet/discussions/1464
+	if (
+		isCanvasBlank(0, 0, bb.w, bb.h, visibleCanvas) ||
+		(extensions.controlNetActive && toolbar._current_tool.name === "Dream")
+	) {
+		//TODO why doesn't smooth rendering toggle persist/localstorage? why am i putting this here? because i'm lazy
+
+		//TODO this logic seems crappy, fix it
+		if (!isCanvasBlank(0, 0, bb.w, bb.h, visibleCanvas)) {
+			// get input image
+			// Temporary canvas for init image and mask generation
+			const bbCanvas = document.createElement("canvas");
+			bbCanvas.width = bb.w;
+			bbCanvas.height = bb.h;
+			const bbCtx = bbCanvas.getContext("2d");
+
+			const maskCanvas = document.createElement("canvas");
+			maskCanvas.width = request.width;
+			maskCanvas.height = request.height;
+			const maskCtx = maskCanvas.getContext("2d");
+
+			const initCanvas = document.createElement("canvas");
+			initCanvas.width = request.width;
+			initCanvas.height = request.height;
+			const initCtx = initCanvas.getContext("2d");
+
+			bbCtx.fillStyle = "#000F";
+
+			// Get init image
+			initCtx.fillRect(0, 0, request.width, request.height);
+			initCtx.drawImage(
+				visibleCanvas,
+				0,
+				0,
+				bb.w,
+				bb.h,
+				0,
+				0,
+				request.width,
+				request.height
+			);
+			// request.init_images = [initCanvas.toDataURL()];
+
+			// Get mask image
+			bbCtx.fillStyle = "#000F";
+			bbCtx.fillRect(0, 0, bb.w, bb.h);
+			if (state.invertMask) {
+				// overmasking by definition is entirely pointless with an inverted mask outpaint
+				// since it should explicitly avoid brushed masks too, we just won't even bother
+				bbCtx.globalCompositeOperation = "destination-in";
+				bbCtx.drawImage(
+					maskPaintCanvas,
+					bb.x,
+					bb.y,
+					bb.w,
+					bb.h,
+					0,
+					0,
+					bb.w,
+					bb.h
+				);
+
+				bbCtx.globalCompositeOperation = "destination-in";
+				bbCtx.drawImage(visibleCanvas, 0, 0);
+			} else {
+				bbCtx.globalCompositeOperation = "destination-in";
+				bbCtx.drawImage(visibleCanvas, 0, 0);
+				// here's where to overmask to avoid including the brushed mask
+				// 99% of my issues were from failing to set source-over for the overmask blotches
+				if (state.overMaskPx > 0) {
+					// transparent to white first
+					bbCtx.globalCompositeOperation = "destination-atop";
+					bbCtx.fillStyle = "#FFFF";
+					bbCtx.fillRect(0, 0, bb.w, bb.h);
+					applyOvermask(bbCanvas, bbCtx, state.overMaskPx);
+				}
+
+				bbCtx.globalCompositeOperation = "destination-out"; // ???
+				bbCtx.drawImage(
+					maskPaintCanvas,
+					bb.x,
+					bb.y,
+					bb.w,
+					bb.h,
+					0,
+					0,
+					bb.w,
+					bb.h
+				);
+			}
+
+			bbCtx.globalCompositeOperation = "destination-atop";
+			bbCtx.fillStyle = "#FFFF";
+			bbCtx.fillRect(0, 0, bb.w, bb.h);
+
+			maskCtx.clearRect(0, 0, maskCanvas.width, maskCanvas.height);
+			maskCtx.drawImage(
+				bbCanvas,
+				0,
+				0,
+				bb.w,
+				bb.h,
+				0,
+				0,
+				request.width,
+				request.height
+			);
+			canvasTransport.initCanvas = initCanvas;
+			canvasTransport.maskCanvas = maskCanvas;
+
+			// getImageAndMask(visibleCanvas, bb, request, state); //FFFFF
+		}
+
+		// request.alwayson_scripts = state.alwayson_scripts;
+
 		if (!global.isOldHRFix && request.enable_hr) {
 			/**
 			 * try and make the new HRfix method useful for our purposes
@@ -1083,6 +1201,31 @@ const dream_generate_callback = async (bb, resolution, state) => {
 				? stableDiffusionData.hr_denoising_strength
 				: 1;
 
+		// add dynamic prompts stuff if it exists because it needs to be explicitly disabled if we don't want it
+		if (extensions.dynamicPromptsEnabled) {
+			addDynamicPromptsToAlwaysOnScripts(state);
+		}
+		if (
+			extensions.controlNetActive &&
+			!isCanvasBlank(0, 0, bb.w, bb.h, visibleCanvas)
+		) {
+			addControlNetToAlwaysOnScripts(
+				state,
+				canvasTransport.initCanvas,
+				canvasTransport.maskCanvas
+			);
+		} else {
+			console.warn(
+				"[dream] controlnet inpaint/outpaint enabled for null image, defaulting to normal txt2img dream"
+			);
+		}
+
+		if (extensions.alwaysOnScripts) {
+			// check again just to be sure because i'm an idiot?
+			// addControlNetToAlwaysOnScripts(state);
+			// addDynamicPromptsToAlwaysOnScripts(state);
+			request.alwayson_scripts = state.alwayson_scripts;
+		}
 		// Dream
 		_generate("txt2img", request, bb);
 	} else {
@@ -1185,10 +1328,25 @@ const dream_generate_callback = async (bb, resolution, state) => {
 			request.width,
 			request.height
 		);
+		// getImageAndMask(visibleCanvas, bb, request, state); // why is not working ffff
 		request.mask = maskCanvas.toDataURL();
 		request.inpainting_fill = stableDiffusionData.outpainting_fill;
 		request.image_cfg_scale = stableDiffusionData.image_cfg_scale;
 
+		// add dynamic prompts stuff if it's enabled
+		if (extensions.dynamicPromptsEnabled) {
+			addDynamicPromptsToAlwaysOnScripts(state);
+		}
+		if (extensions.controlNetActive) {
+			addControlNetToAlwaysOnScripts(state, initCanvas, maskCanvas);
+		}
+		if (extensions.alwaysOnScripts) {
+			// check again just to be sure because i'm an idiot?
+			// addControlNetToAlwaysOnScripts(state);
+			// addDynamicPromptsToAlwaysOnScripts(state);
+			request.alwayson_scripts = state.alwayson_scripts;
+		}
+
 		// Dream
 		_generate("img2img", request, bb, {
 			keepUnmask: state.keepUnmasked ? bbCanvas : null,
@@ -1256,6 +1414,10 @@ const dream_img2img_callback = (bb, resolution, state) => {
 	// Get visible pixels
 	const visibleCanvas = uil.getVisible(bb);
 
+	if (extensions.alwaysOnScripts) {
+		buildAlwaysOnScripts(state);
+	}
+
 	// Do nothing if no image exists
 	if (isCanvasBlank(0, 0, bb.w, bb.h, visibleCanvas)) return;
 
@@ -1400,6 +1562,20 @@ const dream_img2img_callback = (bb, resolution, state) => {
 	request.mask = reqCanvas.toDataURL();
 	request.inpaint_full_res = state.fullResolution;
 
+	// add dynamic prompts stuff if it's enabled
+	if (extensions.dynamicPromptsEnabled) {
+		addDynamicPromptsToAlwaysOnScripts(state);
+	}
+	if (extensions.controlNetActive) {
+		addControlNetToAlwaysOnScripts(state, null, null);
+	}
+	if (extensions.alwaysOnScripts) {
+		// check again just to be sure because i'm an idiot?
+		// addControlNetToAlwaysOnScripts(state);
+		// addDynamicPromptsToAlwaysOnScripts(state);
+		request.alwayson_scripts = state.alwayson_scripts;
+	}
+
 	// Dream
 	_generate("img2img", request, bb, {
 		keepUnmask: state.keepUnmasked ? bbCanvas : null,
@@ -1550,6 +1726,15 @@ const dreamTool = () =>
 					...mouse.coords.world.pos,
 				};
 
+				// state.alwayson_scripts = {};
+				// state.alwayson_scripts.controlnet = {};
+				// state.alwayson_scripts.controlnet.args = [
+				// 	{
+				// 		module: "none",
+				// 		model: "None", //docs have this casing, is that necessary?
+				// 	},
+				// ];
+
 				/**
 				 * Selection handlers
 				 */
@@ -1879,6 +2064,15 @@ const dreamTool = () =>
 						}
 					).checkbox;
 
+					// controlnet checkbox
+					state.ctxmenu.controlNetLabel = _toolbar_input.checkbox(
+						state,
+						"openoutpaint/dream-controlnet",
+						"controlNet",
+						"Toggle ControlNet In/Outpainting",
+						"icon-joystick"
+					).checkbox;
+
 					// Overmasking Slider
 					state.ctxmenu.overMaskPxLabel = _toolbar_input.slider(
 						state,
@@ -1946,6 +2140,10 @@ const dreamTool = () =>
 				//menu.appendChild(document.createElement("br"));
 				array.appendChild(state.ctxmenu.keepUnmaskedLabel);
 				array.appendChild(state.ctxmenu.removeBackgroundLabel);
+				//TODO: if (global.controlnetAPI) { //but figure out how to update the UI after doing so
+				// never mind i think i'm using an extension menu instead
+				// array.appendChild(state.ctxmenu.controlNetLabel);
+				//}
 				menu.appendChild(array);
 				menu.appendChild(state.ctxmenu.keepUnmaskedBlurSlider);
 				menu.appendChild(state.ctxmenu.carveBlurSlider);
@@ -2628,3 +2826,136 @@ const img2imgTool = () =>
 const sendSeed = (seed) => {
 	stableDiffusionData.seed = document.getElementById("seed").value = seed;
 };
+
+function buildAlwaysOnScripts(state) {
+	if (extensions.alwaysOnScripts) {
+		state.alwayson_scripts = {};
+	}
+}
+
+function addDynamicPromptsToAlwaysOnScripts(state) {
+	if (extensions.dynamicPromptsEnabled) {
+		state.alwayson_scripts[extensions.dynamicPromptsAlwaysonScriptName] = {};
+		state.alwayson_scripts[extensions.dynamicPromptsAlwaysonScriptName].args = [
+			extensions.dynamicPromptsActive,
+		];
+	}
+}
+
+function addControlNetToAlwaysOnScripts(state, initCanvas, maskCanvas) {
+	if (extensions.controlNetEnabled && extensions.controlNetActive) {
+		state.alwayson_scripts.controlnet = {};
+		if (initCanvas == null && maskCanvas == null) {
+			//img2img?
+			state.alwayson_scripts.controlnet.args = [
+				{
+					module: extensions.selectedControlNetModule,
+					model: extensions.selectedControlNetModel,
+					control_mode: document.getElementById("controlNetMode-select").value,
+					processor_res: 64,
+					resize_mode: document.getElementById("controlNetResize-select").value,
+					// resize mode?
+					// weights / steps?
+				},
+			];
+		} else {
+			state.alwayson_scripts.controlnet.args = [
+				{
+					module: extensions.selectedControlNetModule,
+					model: extensions.selectedControlNetModel,
+					control_mode: document.getElementById("controlNetMode-select").value,
+					input_image: initCanvas.toDataURL(),
+					mask: maskCanvas.toDataURL(),
+					processor_res: 64,
+					resize_mode: document.getElementById("controlNetResize-select").value,
+					// resize mode?
+					// weights / steps?
+				},
+			];
+		}
+	}
+
+	// 	request.alwayson_scripts = state.alwayson_scripts;
+	// }
+}
+
+// function getImageAndMask(visibleCanvas, bb, request, state) {
+// 	// get input image
+// 	// Temporary canvas for init image and mask generation
+// 	const bbCanvas = document.createElement("canvas");
+// 	bbCanvas.width = bb.w;
+// 	bbCanvas.height = bb.h;
+// 	const bbCtx = bbCanvas.getContext("2d");
+
+// 	const maskCanvas = document.createElement("canvas");
+// 	maskCanvas.width = request.width;
+// 	maskCanvas.height = request.height;
+// 	const maskCtx = maskCanvas.getContext("2d");
+
+// 	const initCanvas = document.createElement("canvas");
+// 	initCanvas.width = request.width;
+// 	initCanvas.height = request.height;
+// 	const initCtx = initCanvas.getContext("2d");
+
+// 	bbCtx.fillStyle = "#000F";
+
+// 	// Get init image
+// 	initCtx.fillRect(0, 0, request.width, request.height);
+// 	initCtx.drawImage(
+// 		visibleCanvas,
+// 		0,
+// 		0,
+// 		bb.w,
+// 		bb.h,
+// 		0,
+// 		0,
+// 		request.width,
+// 		request.height
+// 	);
+// 	// request.init_images = [initCanvas.toDataURL()];
+
+// 	// Get mask image
+// 	bbCtx.fillStyle = "#000F";
+// 	bbCtx.fillRect(0, 0, bb.w, bb.h);
+// 	if (state.invertMask) {
+// 		// overmasking by definition is entirely pointless with an inverted mask outpaint
+// 		// since it should explicitly avoid brushed masks too, we just won't even bother
+// 		bbCtx.globalCompositeOperation = "destination-in";
+// 		bbCtx.drawImage(maskPaintCanvas, bb.x, bb.y, bb.w, bb.h, 0, 0, bb.w, bb.h);
+
+// 		bbCtx.globalCompositeOperation = "destination-in";
+// 		bbCtx.drawImage(visibleCanvas, 0, 0);
+// 	} else {
+// 		bbCtx.globalCompositeOperation = "destination-in";
+// 		bbCtx.drawImage(visibleCanvas, 0, 0);
+// 		// here's where to overmask to avoid including the brushed mask
+// 		// 99% of my issues were from failing to set source-over for the overmask blotches
+// 		if (state.overMaskPx > 0) {
+// 			// transparent to white first
+// 			bbCtx.globalCompositeOperation = "destination-atop";
+// 			bbCtx.fillStyle = "#FFFF";
+// 			bbCtx.fillRect(0, 0, bb.w, bb.h);
+// 			applyOvermask(bbCanvas, bbCtx, state.overMaskPx);
+// 		}
+
+// 		bbCtx.globalCompositeOperation = "destination-out"; // ???
+// 		bbCtx.drawImage(maskPaintCanvas, bb.x, bb.y, bb.w, bb.h, 0, 0, bb.w, bb.h);
+// 	}
+
+// 	bbCtx.globalCompositeOperation = "destination-atop";
+// 	bbCtx.fillStyle = "#FFFF";
+// 	bbCtx.fillRect(0, 0, bb.w, bb.h);
+
+// 	maskCtx.clearRect(0, 0, maskCanvas.width, maskCanvas.height);
+// 	maskCtx.drawImage(
+// 		bbCanvas,
+// 		0,
+// 		0,
+// 		bb.w,
+// 		bb.h,
+// 		0,
+// 		0,
+// 		request.width,
+// 		request.height
+// 	);
+// }

pages/configuration.html

@@ -5,7 +5,7 @@
 		<title>openOutpaint 🐠</title>
 		<!-- CSS Variables -->
 		<link href="../css/colors.css?v=f732f19" rel="stylesheet" />
-		<link href="../css/icons.css?v=a8ec439" rel="stylesheet" />
+		<link href="../css/icons.css?v=e6f94af" rel="stylesheet" />
 
 		<link href="../css/index.css?v=aaad3e5" rel="stylesheet" />
 		<link href="../css/layers.css?v=92c0352" rel="stylesheet" />
@@ -217,6 +217,7 @@
 				localStorage.getItem("openoutpaint/settings.smooth") === null
 					? true
 					: localStorage.getItem("openoutpaint/settings.smooth") === "true";
+			smoothRendering.checked = _enable_smooth;
 
 			writeToLocalStorage();
 

res/icons/joystick.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-joystick"><path d="M21 17a2 2 0 0 0-2-2H5a2 2 0 0 0-2 2v2a2 2 0 0 0 2 2h14a2 2 0 0 0 2-2v-2Z"/><path d="M6 15v-2"/><path d="M12 15V9"/><circle cx="12" cy="6" r="3"/></svg>