adds controlnet reference for probably the wrong reasons

2023-07-15 12:10:45 -05:00 · 2023-07-15 12:10:45 -05:00 · 6461e3b990
commit 6461e3b990
parent a83f72941a
5 changed files with 175 additions and 40 deletions
--- a/README.md
+++ b/README.md
@ -16,6 +16,7 @@ this is a completely vanilla javascript and html canvas outpainting convenience

 ## features

+- SDXL "support"! (please check outpaint/inpaint fill types in the context menus and fiddle with denoising a LOT for img2img, it's touchy)
 - [now available as an extension for webUI!](https://github.com/zero01101/openOutpaint-webUI-extension) you can find it under the default "available" section in the webUI _extensions_ tab
  - **_NOTE: extension still requires `--api` flag in webui-user launch script_**
 - intuitive, convenient outpainting - that's like the whole point right
@ -23,7 +24,8 @@ this is a completely vanilla javascript and html canvas outpainting convenience
 - arbitrary dream reticle size - draw the rectangle of your dreams
 - an [effectively infinite](https://github.com/zero01101/openOutpaint/pull/108), resizable, scalable canvas for you to paint all over
  - **_NOTE: v0.0.10 introduces a new "camera control" modifier key - hold [`CTRL`] and use the scrollwheel to zoom (scroll the wheel or use the two-finger vertical gesture on, uh, modern touchpads) and pan (hold the scrollwheel button, or if you don't have one, left-click button) around the canvas_**
- extremely limited, janky support for a shockingly restrictive list of A1111 extensions including controlnet inpainting for legitimately [magic](https://github.com/Mikubill/sd-webui-controlnet/discussions/1464) [promptless inpainting](https://github.com/Mikubill/sd-webui-controlnet/discussions/1143) and [outpainting](https://github.com/Mikubill/sd-webui-controlnet/discussions/1597)
+- extremely limited, janky support for a shockingly restrictive list of A1111 extensions including controlnet inpainting for legitimately [magic](https://github.com/Mikubill/sd-webui-controlnet/discussions/1464) [promptless inpainting](https://github.com/Mikubill/sd-webui-controlnet/discussions/1143) and [outpainting](https://github.com/Mikubill/sd-webui-controlnet/discussions/1597) and in-line reference preprocessors for keeping existing style while replacing things (reference requires at least 2 controlnet units enabled in A1111 settings), as well as a very very basic dynamic-prompts-on-or-off toggle
+  - **_NOTE: this is_ JANKY, _pull requests greatly welcomed lol_**
 - a very nicely functional and familiar layer system
 - save, load, import, and export workspaces - includes all your layers, history, canvas size, you name it!
 - inpainting/touchup mask brush
--- a/index.html
+++ b/index.html
@ -1,4 +1,4 @@
-<!doctype html>
+<!DOCTYPE html>
 <html lang="en-US">
 	<head>
 		<meta charset="utf-8" />
@ -213,31 +213,59 @@
 						onchange="changeControlNetExtension()"
 						disabled="disabled" />
 					<label for="cbxControlNet">ControlNet In/Outpainting</label>
-					<br class="controlnetElement" />
-					<label id="cnModuleLabel" class="controlnetElement">
-						Preprocessor
+					<br class="controlNetElement" />
+					<label id="cnModuleLabel" class="controlNetElement">
+						Inpaint Preprocessor
 					</label>
-					<div id="controlNetModule-ac-select" class="controlnetElement"></div>
-					<label id="cnModelLabel" class="controlnetElement">Model</label>
-					<div id="controlNetModel-ac-select" class="controlnetElement"></div>
-					<label id="cnControlLabel" class="controlnetElement">
-						Control Mode
+					<div id="controlNetModule-ac-select" class="controlNetElement"></div>
+					<label id="cnModelLabel" class="controlNetElement">Model</label>
+					<div id="controlNetModel-ac-select" class="controlNetElement"></div>
+					<label id="cnControlLabel" class="controlNetElement">
+						Inpaint Mode
 					</label>
-					<select id="controlNetMode-select" class="controlnetElement">
+					<select id="controlNetMode-select" class="controlNetElement">
 						<option value="Balanced">balanced</option>
 						<option value="My prompt is more important">+prompt</option>
 						<option value="ControlNet is more important">+CN</option>
 					</select>
-					<br />
-					<label id="cnResizeLabel" class="controlnetElement">
+					<br class="controlNetElement" />
+					<label id="cnResizeLabel" class="controlNetElement">
 						Resize Mode
 					</label>
-					<select id="controlNetResize-select" class="controlnetElement">
+					<select id="controlNetResize-select" class="controlNetElement">
 						<option value="Just Resize">resize</option>
 						<option value="Crop and Resize">+crop</option>
 						<option value="Resize and Fill">+fill</option>
 					</select>
-					<!-- <div id="referenceStyleFidelity" class="controlnetElement"></div> -->
+					<br class="controlNetElement" />
+					<input
+						type="checkbox"
+						id="cbxControlNetReferenceLayer"
+						onchange="changeControlNetReference()"
+						class="controlNetElement" />
+					<label for="cbxControlNetReferenceLayer" class="controlNetElement">
+						Reference as 1st CN Unit
+					</label>
+					<br class="controlNetReferenceElement" />
+					<label id="cnReferenceModuleLabel" class="controlNetReferenceElement">
+						Reference Preprocessor
+					</label>
+					<div
+						id="controlNetReferenceModule-ac-select"
+						class="controlNetReferenceElement"></div>
+					<div
+						id="controlNetReferenceFidelity"
+						class="controlNetReferenceElement"></div>
+					<label id="cnResizeLabel" class="controlNetReferenceElement">
+						Reference Mode
+					</label>
+					<select
+						id="controlNetReferenceMode-select"
+						class="controlNetReferenceElement">
+						<option value="Balanced">balanced</option>
+						<option value="My prompt is more important">+prompt</option>
+						<option value="ControlNet is more important">+CN</option>
+					</select>
 				</div>
 				<!-- Save/load image section -->
 				<button type="button" class="collapsible">Save/Upscaling</button>
@ -287,8 +315,8 @@
 					<br />
 					<span id="version">
 						<a href="https://github.com/zero01101/openOutpaint" target="_blank">
-							<s>Alpha release v0.0.16.2</s>
-							v20230715.001
+							<s>Alpha release v0.0.16.3</s>
+							v20230715.002
 						</a>
 						<br />
 						<a
@ -485,7 +513,7 @@
 		<!-- Basics -->
 		<script src="js/global.js?v=ac30d16" type="text/javascript"></script>
 		<script src="js/defaults.js?v=5b06818" type="text/javascript"></script>
-		<script src="js/extensions.js?v=abb882a" type="text/javascript"></script>
+		<script src="js/extensions.js?v=1fca0fc" type="text/javascript"></script>

 		<!-- Base Libs -->
 		<script src="js/lib/util.js?v=379aef7" type="text/javascript"></script>
@ -511,7 +539,7 @@

 		<!-- Content -->
 		<script src="js/prompt.js?v=7a1c68c" type="text/javascript"></script>
-		<script src="js/index.js?v=31d52e7" type="text/javascript"></script>
+		<script src="js/index.js?v=7debf82" type="text/javascript"></script>

 		<script
 			src="js/ui/floating/history.js?v=4f29db4"
@ -525,7 +553,7 @@
 			src="js/ui/tool/generic.js?v=3e678e0"
 			type="text/javascript"></script>

-		<script src="js/ui/tool/dream.js?v=ac1643d" type="text/javascript"></script>
+		<script src="js/ui/tool/dream.js?v=06e0164" type="text/javascript"></script>
 		<script
 			src="js/ui/tool/maskbrush.js?v=e9bd0eb"
 			type="text/javascript"></script>
--- a/js/extensions.js
+++ b/js/extensions.js
@ -7,8 +7,12 @@ const extensions = {
 	alwaysOnScripts: false,
 	controlNetEnabled: false,
 	controlNetActive: false,
+	controlNetReferenceActive: false,
+	controlNetReferenceFidelity: 0.5,
 	selectedControlNetModel: null,
 	selectedControlNetModule: null,
+	selectedCNReferenceModule: null,
+	controlNetModelCount: 0,
 	dynamicPromptsEnabled: false,
 	dynamicPromptsActive: false,
 	dynamicPromptsAlwaysonScriptName: null, //GRUMBLE GRUMBLE
@ -18,7 +22,8 @@ const extensions = {

 	async getExtensions(
 		controlNetModelAutoComplete,
-		controlNetModuleAutoComplete
+		controlNetModuleAutoComplete,
+		controlNetReferenceModuleAutoComplete
 	) {
 		const allowedExtensions = [
 			"controlnet",
@ -51,7 +56,8 @@ const extensions = {
 		this.checkForDynamicPrompts();
 		this.checkForControlNet(
 			controlNetModelAutoComplete,
-			controlNetModuleAutoComplete
+			controlNetModuleAutoComplete,
+			controlNetReferenceModuleAutoComplete
 		);
 		//checkForSAM(); //or inpaintAnything or something i dunno
 		//checkForADetailer(); //? this one seems iffy
@ -78,7 +84,8 @@ const extensions = {

 	async checkForControlNet(
 		controlNetModelAutoComplete,
-		controlNetModuleAutoComplete
+		controlNetModuleAutoComplete,
+		controlNetReferenceModuleAutoComplete
 	) {
 		var url = document.getElementById("host").value + "/controlnet/version";

@ -97,8 +104,23 @@ const extensions = {
 				document.getElementById("cbxControlNet").disabled = false;
 				// ok cool so now we can get the models and modules
 				this.getModels(controlNetModelAutoComplete);
-				this.getModules(controlNetModuleAutoComplete);
+				this.getModules(
+					controlNetModuleAutoComplete,
+					controlNetReferenceModuleAutoComplete
+				);
 			}
+			url = document.getElementById("host").value + "/controlnet/settings";
+			try {
+				const response2 = await fetch(url);
+				const data2 = await response2.json();
+				if (data2.control_net_max_models_num < 2) {
+					document.getElementById("cbxControlNetReferenceLayer").disabled =
+						"disabled";
+					console.warn(
+						"[extensions] ControlNet reference layer disabled due to insufficient units enabled in settings - cannot be enabled via API, please increase to at least 2 units manually"
+					);
+				}
+			} catch (ex) {}
 		} catch (e) {
 			// ??
 			global.controlnetAPI = false;
@ -128,7 +150,10 @@ const extensions = {

 		controlNetModelAutoComplete.options = opt;
 	},
-	async getModules(controlNetModuleAutoComplete) {
+	async getModules(
+		controlNetModuleAutoComplete,
+		controlNetReferenceModuleAutoComplete
+	) {
 		const allowedModules = ["reference", "inpaint"];
 		var url = document.getElementById("host").value + "/controlnet/module_list";

@ -156,5 +181,14 @@ const extensions = {
 		});

 		controlNetModuleAutoComplete.options = opt;
+
+		opt = this.controlNetModules.module_list
+			.filter((m) => m.includes("reference"))
+			.map((option) => ({
+				name: option,
+				value: option,
+			}));
+
+		controlNetReferenceModuleAutoComplete.options = opt;
 	},
 };
--- a/js/index.js
+++ b/js/index.js
@ -172,6 +172,7 @@ function startup() {
 	changeRestoreFaces();
 	changeSyncCursorSize();
 	changeControlNetExtension();
+	changeControlNetReference();
 	checkFocus();
 	refreshScripts();
 }
@ -424,7 +425,8 @@ async function testHostConnection() {
 							getModels();
 							extensions.getExtensions(
 								controlNetModelAutoComplete,
-								controlNetModuleAutoComplete
+								controlNetModuleAutoComplete,
+								controlNetReferenceModuleAutoComplete
 							);
 							getLoras();
 							// getTIEmbeddings();
@ -680,7 +682,7 @@ const hrFixUpscalerAutoComplete = createAutoComplete(
 );

 let controlNetModelAutoComplete = createAutoComplete(
-	"ControlNet Model",
+	"Inpaint Model",
 	document.getElementById("controlNetModel-ac-select")
 );

@ -689,7 +691,7 @@ controlNetModelAutoComplete.onchange.on(({value}) => {
 });

 let controlNetModuleAutoComplete = createAutoComplete(
-	"ControlNet Module",
+	"Inpaint Preprocessor",
 	document.getElementById("controlNetModule-ac-select")
 );

@ -697,6 +699,15 @@ controlNetModuleAutoComplete.onchange.on(({value}) => {
 	extensions.selectedControlNetModule = value;
 });

+let controlNetReferenceModuleAutoComplete = createAutoComplete(
+	"Reference Preprocessor",
+	document.getElementById("controlNetReferenceModule-ac-select")
+);
+
+controlNetReferenceModuleAutoComplete.onchange.on(({value}) => {
+	extensions.selectedCNReferenceModule = value;
+});
+
 // const extensionsAutoComplete = createAutoComplete(
 // 	"Extension",
 // 	document.getElementById("extension-ac-select")
@ -724,6 +735,25 @@ const resSlider = makeSlider(
 			toolbar.currentTool.redraw();
 	}
 );
+
+const refSlider = makeSlider(
+	"Reference Fidelity",
+	document.getElementById("controlNetReferenceFidelity"),
+	"cn_reference_fidelity",
+	0.0,
+	1.0,
+	0.1,
+	0.5,
+	0.01,
+	(v) => {
+		extensions.controlNetReferenceFidelity = v;
+
+		toolbar.currentTool &&
+			toolbar.currentTool.redraw &&
+			toolbar.currentTool.redraw();
+	}
+);
+
 makeSlider(
 	"CFG Scale",
 	document.getElementById("cfgScale"),
@ -851,11 +881,26 @@ function changeControlNetExtension() {
 		document.getElementById("cbxControlNet").checked;
 	if (extensions.controlNetActive) {
 		document
-			.querySelectorAll(".controlnetElement")
+			.querySelectorAll(".controlNetElement")
 			.forEach((el) => el.classList.remove("invisible"));
 	} else {
 		document
-			.querySelectorAll(".controlnetElement")
+			.querySelectorAll(".controlNetElement")
+			.forEach((el) => el.classList.add("invisible"));
+	}
+}
+
+function changeControlNetReference() {
+	extensions.controlNetReferenceActive = document.getElementById(
+		"cbxControlNetReferenceLayer"
+	).checked;
+	if (extensions.controlNetReferenceActive) {
+		document
+			.querySelectorAll(".controlNetReferenceElement")
+			.forEach((el) => el.classList.remove("invisible"));
+	} else {
+		document
+			.querySelectorAll(".controlNetReferenceElement")
 			.forEach((el) => el.classList.add("invisible"));
 	}
 }
--- a/js/ui/tool/dream.js
+++ b/js/ui/tool/dream.js
@ -62,12 +62,9 @@ const _monitorProgress = (bb, oncheck = null) => {
 		}

 		const timeSpent = performance.now() - init;
-		setTimeout(
-			() => {
+		setTimeout(() => {
 			if (running) _checkProgress();
-			},
-			Math.max(0, minDelay - timeSpent)
-		);
+		}, Math.max(0, minDelay - timeSpent));
 	};

 	_checkProgress();
@ -1217,7 +1214,7 @@ const dream_generate_callback = async (bb, resolution, state) => {
 				canvasTransport.initCanvas,
 				canvasTransport.maskCanvas
 			);
-		} else {
+		} else if (extensions.controlNetActive) {
 			console.warn(
 				"[dream] controlnet inpaint/outpaint enabled for null image, defaulting to normal txt2img dream"
 			);
@ -1570,7 +1567,15 @@ const dream_img2img_callback = (bb, resolution, state) => {
 		addDynamicPromptsToAlwaysOnScripts(state);
 	}
 	if (extensions.controlNetActive) {
-		addControlNetToAlwaysOnScripts(state, null, null);
+		if (extensions.controlNetReferenceActive) {
+			addControlNetToAlwaysOnScripts(
+				state,
+				request.init_images[0],
+				request.mask
+			);
+		} else {
+			addControlNetToAlwaysOnScripts(state, null, null); // //WTF???
+		}
 	}
 	if (extensions.alwaysOnScripts) {
 		// check again just to be sure because i'm an idiot?
@ -2846,6 +2851,10 @@ function addDynamicPromptsToAlwaysOnScripts(state) {
 }

 function addControlNetToAlwaysOnScripts(state, initCanvas, maskCanvas) {
+	var initimg =
+		toolbar._current_tool.name == "Dream" ? initCanvas.toDataURL() : initCanvas;
+	var maskimg =
+		toolbar._current_tool.name == "Dream" ? maskCanvas.toDataURL() : maskCanvas;
 	if (extensions.controlNetEnabled && extensions.controlNetActive) {
 		state.alwayson_scripts.controlnet = {};
 		if (initCanvas == null && maskCanvas == null) {
@ -2867,8 +2876,8 @@ function addControlNetToAlwaysOnScripts(state, initCanvas, maskCanvas) {
 					module: extensions.selectedControlNetModule,
 					model: extensions.selectedControlNetModel,
 					control_mode: document.getElementById("controlNetMode-select").value,
-					input_image: initCanvas.toDataURL(),
-					mask: maskCanvas.toDataURL(),
+					input_image: initimg, //initCanvas.toDataURL(),
+					mask: maskimg, //maskCanvas.toDataURL(),
 					processor_res: 64,
 					resize_mode: document.getElementById("controlNetResize-select").value,
 					// resize mode?
@ -2876,7 +2885,24 @@ function addControlNetToAlwaysOnScripts(state, initCanvas, maskCanvas) {
 				},
 			];
 		}
+		if (extensions.controlNetReferenceActive) {
+			state.alwayson_scripts.controlnet.args.unshift({
+				enabled: true,
+				module: extensions.selectedCNReferenceModule,
+				model: "None",
+				control_mode: document.getElementById("controlNetReferenceMode-select")
+					.value,
+				image: initimg, //initCanvas.toDataURL(),
+				processor_res: 64,
+				threshold_a: extensions.controlNetReferenceFidelity,
+				threshold_b: 64,
+				resize_mode: document.getElementById("controlNetResize-select").value,
+			});
 		}
+	}
+
+	var deleteme = 2463;
+	var ok = deleteme / 36;

 	// 	request.alwayson_scripts = state.alwayson_scripts;
 	// }