From 3d45e549c014a59fc34bc9f950e1491f6a5ce19d Mon Sep 17 00:00:00 2001 From: tim h Date: Thu, 17 Nov 2022 21:21:48 -0600 Subject: [PATCH] remedial overmasking FINALLY --- .gitignore | 1 + README.md | 15 +++- css/index.css | 11 ++- index.html | 17 ++++- js/index.js | 204 ++++++++++++++++++++++++++++++++++++++++++++------ 5 files changed, 216 insertions(+), 32 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a3062be --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.vscode/* diff --git a/README.md b/README.md index c17a25e..145dba6 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,14 @@ this is a completely vanilla javascript and html canvas outpainting convenience - i am terrible at javascript and should probably correct that - i have never used html canvas for anything before and should try it out + ## features + - a big ol' 2560x1440 canvas for you to paint all over _(infinite canvas area planned, in //todo already)_ + - inpainting/touchup blob + - easily change samplers/steps/CFG/etc options for each "dream" summoned from the latent void + - optional grid snapping for precision + - optional overmasking for better seams between outpaints (suggested by @lifeh2o ([a](https://www.reddit.com/r/StableDiffusion/comments/ywf8np/i_made_a_completely_local_offline_opensource/iwl6s06/),[b](https://www.reddit.com/r/StableDiffusion/comments/ys9lhq/kollai_an_infinite_multiuser_canvas_running_on/ivzygwk/?context=3)) and i think it's a slick idea) + - "temporary" monitors at the bottom to see exactly what mask/image you're feeding img2img, no i'm certainly not using them as actual imagedata sources or anything + ## operation ### prerequisities @@ -47,13 +55,13 @@ this is a completely vanilla javascript and html canvas outpainting convenience ### in order of "priority"/likelihood of me doing it - [ ] lots and lots of readme updates (ongoing) - [ ] comment basically everything that isn't self documenting (ongoing) -- [ ] _CURRENT TASK_: overmask seam of img2img (https://www.reddit.com/r/StableDiffusion/comments/ys9lhq/kollai_an_infinite_multiuser_canvas_running_on/ivzygwk/?context=3) +- [x] overmask seam of img2img - [x] split out CSS to its own file (remedial cleanup task) -- [ ] split out JS to separation-of-concerns individual files (oh no) -- [ ] ability to blank/new canvas without making the user refresh because that's pretty janky +- [ ] ability to blank/new canvas without making the user refresh the page because that's pretty janky - [ ] add error handling for async/XHR POST in case of, yknow, errors - [ ] image erase region in case you decide later that you're not too happy with earlier results (technically i guess you could just mask over the entire region you dislike but that's... bad) - [ ] controls for the rest of API-available options (e.g. hires fix, inpaint fill modes, etc) +- [ ] save user-set option values to browser localstorage to persist your preferred, uh, preferences - [ ] render progress spinner/bar - [ ] ~~smart crop downloaded image~~ - [ ] import external image and scale/superimpose at will on canvas for in/outpainting @@ -65,6 +73,7 @@ this is a completely vanilla javascript and html canvas outpainting convenience - [ ] infinite canvas - [ ] global undo/redo - [ ] inpainting sketch tools +- [ ] split out JS to separation-of-concerns individual files (oh no) - [ ] something actually similar to a "user interface", preferably visually pleasant and would make my mom say "well that makes sense" if she looked at it - [ ] eventually delete the generated mask display canvases at the bottom of the page, but they're useful for debugging canvas pixel offsets sometimes - [ ] see if i can use fewer canvases overall; seems wasteful, canvas isn't free yknow diff --git a/css/index.css b/css/index.css index 01e4014..cb551e8 100644 --- a/css/index.css +++ b/css/index.css @@ -58,17 +58,22 @@ .masks { display: grid; - grid-template-columns: repeat(2, 1fr); + grid-template-columns: repeat(3, 1fr); grid-template-rows: 1fr; grid-column-gap: 0px; grid-row-gap: 0px; } -.maskCanvas { +.maskCanvasMonitor { position: absolute; } -.initImgCanvas { +.overMaskCanvasMonitor { position: absolute; left: 600px; +} + +.initImgCanvasMonitor { + position: absolute; + left: 1200px; } \ No newline at end of file diff --git a/index.html b/index.html index 3fb82c1..b745e7b 100644 --- a/index.html +++ b/index.html @@ -43,12 +43,20 @@
+ +



+ +
+ +


@@ -156,10 +164,13 @@
- +

lol ur browser sucks

 - + +

lol ur browser sucks

+ +

lol ur browser sucks
diff --git a/js/index.js b/js/index.js index 34e3d72..31625d5 100644 --- a/js/index.js +++ b/js/index.js @@ -75,7 +75,9 @@ var backupMaskChunk = null; var backupMaskX = null; var backupMaskY = null; var totalImagesReturned; - +var maskEdgePixels = {}; +var overMask = true; //TODO need toggle IMMEDIATELY for this +var overMaskPx = 10; //TODO need control IMMEDIATELY for this... once it works.... var drawTargets = []; // is this needed? i only draw the last one anyway... // info div, sometimes hidden @@ -112,6 +114,9 @@ function startup() { changeBatchSize(); changeSnapMode(); changeMaskBlur(); + changeSeed(); + changeOverMask(); + changeOverMaskPx(); document.getElementById("overlayCanvas").onmousemove = mouseMove; document.getElementById("overlayCanvas").onmousedown = mouseDown; document.getElementById("overlayCanvas").onmouseup = mouseUp; @@ -383,24 +388,39 @@ function mouseUp(evt) { // console.log(downX + ":" + downY + " :: " + this.isCanvasBlank(downX, downY)); if (!isCanvasBlank(drawIt.x, drawIt.y, drawIt.w, drawIt.h, imgCanvas)) { // img2img - var ctx = document.getElementById("canvas").getContext("2d"); - const imgChunk = ctx.getImageData(drawIt.x, drawIt.y, drawIt.w, drawIt.h); - const imgChunkData = imgChunk.data; - var canvas2 = document.getElementById("maskCanvas"); - var ctx2 = canvas2.getContext("2d"); - var canvas3 = document.getElementById("initImgCanvas"); - var ctx3 = canvas3.getContext("2d"); + var mainCanvasCtx = document.getElementById("canvas").getContext("2d"); + const imgChunk = mainCanvasCtx.getImageData(drawIt.x, drawIt.y, drawIt.w, drawIt.h); // imagedata object of the image being outpainted + const imgChunkData = imgChunk.data; // imagedata.data object, a big inconvenient uint8clampedarray + // these are the 3 mask monitors on the bottom of the page + var maskCanvas = document.getElementById("maskCanvasMonitor"); + var maskCanvasCtx = maskCanvas.getContext("2d"); + var initImgCanvas = document.getElementById("initImgCanvasMonitor"); + var initImgCanvasCtx = initImgCanvas.getContext("2d"); + var overMaskCanvas = document.getElementById("overMaskCanvasMonitor"); + var overMaskCanvasCtx = overMaskCanvas.getContext("2d"); // get blank pixels to use as mask - const maskImgData = ctx2.createImageData(drawIt.w, drawIt.h); - const initImgData = ctx2.createImageData(drawIt.w, drawIt.h); + const maskImgData = maskCanvasCtx.createImageData(drawIt.w, drawIt.h); + const initImgData = mainCanvasCtx.createImageData(drawIt.w, drawIt.h); + const overMaskImgData = overMaskCanvasCtx.createImageData(drawIt.w, drawIt.h); + // cover entire masks in black before adding masked areas + for (let i = 0; i < imgChunkData.length; i += 4) { // l->r, top->bottom, R G B A pixel values in a big ol array - // make a simple mask + // can i log the x,y of pixels that are transparent so i can easily just add different pixels and get the index via https://stackoverflow.com/questions/45963306/html5-canvas-how-to-get-adjacent-pixels-position-from-the-linearized-imagedata/45969661#45969661 ? + + // make a simple mask if (imgChunkData[i + 3] == 0) { // rgba pixel values, 4th one is alpha, if it's 0 there's "nothing there" in the image display canvas and its time to outpaint maskImgData.data[i] = 255; // white mask gets painted over maskImgData.data[i + 1] = 255; maskImgData.data[i + 2] = 255; maskImgData.data[i + 3] = 255; + + overMaskImgData.data[i] = 255; //lets just set this up now + overMaskImgData.data[i + 1] = 255; + overMaskImgData.data[i + 2] = 255; + overMaskImgData.data[i + 3] = 255; + + initImgData.data[i] = 0; // null area on initial image becomes opaque black pixels initImgData.data[i + 1] = 0; initImgData.data[i + 2] = 0; @@ -410,12 +430,131 @@ function mouseUp(evt) { maskImgData.data[i + 1] = 0; maskImgData.data[i + 2] = 0; maskImgData.data[i + 3] = 255; // but it still needs an opaque alpha channel + + overMaskImgData.data[i] = 0; + overMaskImgData.data[i + 1] = 0; + overMaskImgData.data[i + 2] = 0; + overMaskImgData.data[i + 3] = 255; + initImgData.data[i] = imgChunkData[i]; // put the original picture back in the painted area initImgData.data[i + 1] = imgChunkData[i + 1]; initImgData.data[i + 2] = imgChunkData[i + 2]; initImgData.data[i + 3] = imgChunkData[i + 3]; //it's still RGBA so we can handily do this in nice chunks'o'4 } } + + + // make a list of all the white pixels to expand so we don't waste time on non-mask pixels + let pix = { x: [], y: [], index: [] }; + var x, y, index; + for (y = 0; y < drawIt.h; y++) { + for (x = 0; x < drawIt.w; x++) { + index = ((y * drawIt.w + x) * 4); + if (overMaskImgData.data[index] > 0) { + pix.x.push(x); + pix.y.push(y); + pix.index.push(index); + } + } + } + + for (i = 0; i < pix.index.length; i++) { + // get the index in the stupid array + var currentMaskPixelIndex = pix.index[i]; + + // for any horizontal expansion, we need to ensure that the target pixel is in the same Y row + // horizontal left (west) is index-4 per pixel + // horizontal right (east) is index+4 per pixel + var currentMaskPixelY = pix.y[i]; + + // for any vertical expansion, we need to ensure that the target pixel is in the same X column + // vertical up (north) is index-(imagedata.width) per pixel + // vertical down (south) is index+(imagedata.width) per pixel + var currentMaskPixelX = pix.x[i]; + + // i hate uint8clampedarray and math + // primarily math + // actually just my brain + // ok so now lets check neighbors to see if they're in the same row/column + for (j = overMaskPx; j > 0; j--) { // set a variable to the extreme end of the overmask size and work our way back inwards + // i hate uint8clampedarray and math + // this is so inefficient but i warned you all i'm bad at this + //TODO refactor like all of this, it's horrible and shameful + // BUT IT WORKS + // but it is crushingly inefficient i'm sure + // BUT IT WORKS and i came up with it all by myself because i'm a big boy + + // west + var potentialPixelIndex = ((currentMaskPixelY * drawIt.w + currentMaskPixelX) * 4) - (j * 4); + var potentialPixelX = (potentialPixelIndex / 4) % drawIt.w; + var potentialPixelY = Math.floor((potentialPixelIndex / 4) / drawIt.w); + // ENSURE SAME ROW using the y axis unintuitively + if (potentialPixelY == currentMaskPixelY) { + // ok then + // ensure it's not already a mask pixel + if (overMaskImgData.data[potentialPixelIndex] != 255) { + // welp fingers crossed + overMaskImgData.data[potentialPixelIndex] = 255; + overMaskImgData.data[potentialPixelIndex + 1] = 255; + overMaskImgData.data[potentialPixelIndex + 2] = 255; + overMaskImgData.data[potentialPixelIndex + 3] = 255; + } + } + + // east + var potentialPixelIndex = ((currentMaskPixelY * drawIt.w + currentMaskPixelX) * 4) + (j * 4); + var potentialPixelX = (potentialPixelIndex / 4) % drawIt.w; + var potentialPixelY = Math.floor((potentialPixelIndex / 4) / drawIt.w); + // ENSURE SAME ROW using the y axis unintuitively + if (potentialPixelY == currentMaskPixelY) { + // ok then + // ensure it's not already a mask pixel + if (overMaskImgData.data[potentialPixelIndex] != 255) { + // welp fingers crossed + overMaskImgData.data[potentialPixelIndex] = 255; + overMaskImgData.data[potentialPixelIndex + 1] = 255; + overMaskImgData.data[potentialPixelIndex + 2] = 255; + overMaskImgData.data[potentialPixelIndex + 3] = 255; + } + } + + // north + var potentialPixelIndex = ((currentMaskPixelY * drawIt.w + currentMaskPixelX) * 4) - ((j * drawIt.w) * 4); + var potentialPixelX = (potentialPixelIndex / 4) % drawIt.w; + var potentialPixelY = Math.floor((potentialPixelIndex / 4) / drawIt.w); + // ENSURE SAME COLUMN using the x axis unintuitively + if (potentialPixelX == currentMaskPixelX) { + // ok then + // ensure it's not already a mask pixel + if (overMaskImgData.data[potentialPixelIndex] != 255) { + // welp fingers crossed + overMaskImgData.data[potentialPixelIndex] = 255; + overMaskImgData.data[potentialPixelIndex + 1] = 255; + overMaskImgData.data[potentialPixelIndex + 2] = 255; + overMaskImgData.data[potentialPixelIndex + 3] = 255; + } + } + + // south + var potentialPixelIndex = ((currentMaskPixelY * drawIt.w + currentMaskPixelX) * 4) + ((j * drawIt.w) * 4); + var potentialPixelX = (potentialPixelIndex / 4) % drawIt.w; + var potentialPixelY = Math.floor((potentialPixelIndex / 4) / drawIt.w); + // ENSURE SAME COLUMN using the x axis unintuitively + if (potentialPixelX == currentMaskPixelX) { + // ok then + // ensure it's not already a mask pixel + if (overMaskImgData.data[potentialPixelIndex] != 255) { + // welp fingers crossed + overMaskImgData.data[potentialPixelIndex] = 255; + overMaskImgData.data[potentialPixelIndex + 1] = 255; + overMaskImgData.data[potentialPixelIndex + 2] = 255; + overMaskImgData.data[potentialPixelIndex + 3] = 255; + } + } + } + + } + // also check for painted masks in region, add them as white pixels to mask canvas const maskChunk = maskPaintCtx.getImageData(drawIt.x, drawIt.y, drawIt.w, drawIt.h); const maskChunkData = maskChunk.data; @@ -425,6 +564,10 @@ function mouseUp(evt) { maskImgData.data[i + 1] = 255; maskImgData.data[i + 2] = 255; maskImgData.data[i + 3] = 255; + overMaskImgData.data[i] = 255; + overMaskImgData.data[i + 1] = 255; + overMaskImgData.data[i + 2] = 255; + overMaskImgData.data[i + 3] = 255; } } // backup any painted masks ingested then them, replacable if user doesn't like resultant image @@ -439,13 +582,17 @@ function mouseUp(evt) { } maskPaintCtx.putImageData(clearArea, drawIt.x, drawIt.y); // mask monitors - ctx2.putImageData(maskImgData, 0, 0); - var maskBase64 = canvas2.toDataURL(); - ctx3.putImageData(initImgData, 0, 0); - var initImgBase64 = canvas3.toDataURL(); + maskCanvasCtx.putImageData(maskImgData, 0, 0); + var maskBase64 = maskCanvas.toDataURL(); + overMaskCanvasCtx.putImageData(overMaskImgData, 0, 0); // :pray: + var overMaskBase64 = overMaskCanvas.toDataURL(); + initImgCanvasCtx.putImageData(initImgData, 0, 0); + var initImgBase64 = initImgCanvas.toDataURL(); // img2img endpoint = "img2img"; - stableDiffusionData.mask = maskBase64; + var selectedMask = overMask ? overMaskBase64 : maskBase64; + stableDiffusionData.mask = selectedMask; + // stableDiffusionData.mask = maskBase64; stableDiffusionData.init_images = [initImgBase64]; // slightly more involved than txt2img } else { @@ -455,14 +602,8 @@ function mouseUp(evt) { } stableDiffusionData.prompt = document.getElementById("prompt").value; stableDiffusionData.negative_prompt = document.getElementById("negPrompt").value; - // stableDiffusionData.sampler_index = sampler; - // stableDiffusionData.steps = steps; - // stableDiffusionData.cfg_scale = cfgScale; stableDiffusionData.width = drawIt.w; stableDiffusionData.height = drawIt.h; - // stableDiffusionData.batch_size = batchSize; - // stableDiffusionData.n_iter = batchCount; - // stableDiffusionData.mask_blur = maskBlur; dream(drawIt.x, drawIt.y, stableDiffusionData); } } @@ -512,6 +653,18 @@ function changeMaskBlur() { stableDiffusionData.mask_blur = document.getElementById("maskBlur").value; } +function changeSeed() { + stableDiffusionData.seed = document.getElementById("seed").value; +} + +function changeOverMask() { + overMask = document.getElementById("cbxOverMask").checked; +} + +function changeOverMaskPx() { + overMaskPx = document.getElementById("overMaskPx").value; +} + function isCanvasBlank(x, y, w, h, specifiedCanvas) { var canvas = document.getElementById(specifiedCanvas.id); return !canvas.getContext('2d') @@ -554,18 +707,23 @@ function cropCanvas(sourceCanvas) { for (y = 0; y < h; y++) { for (x = 0; x < w; x++) { - index = (y * w + x) * 4; + // lol i need to learn what this part does + index = (y * w + x) * 4; // OHHH OK this is setting the imagedata.data uint8clampeddataarray index for the specified x/y coords + //this part i get, this is checking that 4th RGBA byte for opacity if (imageData.data[index + 3] > 0) { pix.x.push(x); pix.y.push(y); } } } + // ...need to learn what this part does too :badpokerface: + // is this just determining the boundaries of non-transparent pixel data? pix.x.sort(function (a, b) { return a - b }); pix.y.sort(function (a, b) { return a - b }); var n = pix.x.length - 1; w = pix.x[n] - pix.x[0]; h = pix.y[n] - pix.y[0]; + // yup sure looks like it try { var cut = sourceCanvas.getContext('2d').getImageData(pix.x[0], pix.y[0], w, h);