Added bluring to mask for flex2
This commit is contained in:
1 parent
77763a3e5c
commit
ac1ee559c5
1 file changed
+34
-2
@@ -2,6 +2,7 @@ import os
|
|||||||
from typing import TYPE_CHECKING, List
|
from typing import TYPE_CHECKING, List
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
import torchvision
|
||||||
import yaml
|
import yaml
|
||||||
from toolkit import train_tools
|
from toolkit import train_tools
|
||||||
from toolkit.config_modules import GenerateImageConfig, ModelConfig
|
from toolkit.config_modules import GenerateImageConfig, ModelConfig
|
||||||
@@ -37,6 +38,15 @@ scheduler_config = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def random_blur(img, min_kernel_size=3, max_kernel_size=23, p=0.5):
|
||||||
|
if random.random() < p:
|
||||||
|
kernel_size = random.randint(min_kernel_size, max_kernel_size)
|
||||||
|
# make sure it is odd
|
||||||
|
if kernel_size % 2 == 0:
|
||||||
|
kernel_size += 1
|
||||||
|
img = torchvision.transforms.functional.gaussian_blur(img, kernel_size=kernel_size)
|
||||||
|
return img
|
||||||
|
|
||||||
class Flex2(BaseModel):
|
class Flex2(BaseModel):
|
||||||
arch = "flex2"
|
arch = "flex2"
|
||||||
|
|
||||||
@@ -66,6 +76,8 @@ class Flex2(BaseModel):
|
|||||||
self.inpaint_dropout = model_config.model_kwargs.get('inpaint_dropout', 0.0)
|
self.inpaint_dropout = model_config.model_kwargs.get('inpaint_dropout', 0.0)
|
||||||
self.control_dropout = model_config.model_kwargs.get('control_dropout', 0.0)
|
self.control_dropout = model_config.model_kwargs.get('control_dropout', 0.0)
|
||||||
self.inpaint_random_chance = model_config.model_kwargs.get('inpaint_random_chance', 0.0)
|
self.inpaint_random_chance = model_config.model_kwargs.get('inpaint_random_chance', 0.0)
|
||||||
|
self.random_blur_mask = model_config.model_kwargs.get('random_blur_mask', False)
|
||||||
|
self.do_random_inpainting = model_config.model_kwargs.get('do_random_inpainting', False)
|
||||||
|
|
||||||
# static method to get the noise scheduler
|
# static method to get the noise scheduler
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -370,13 +382,17 @@ class Flex2(BaseModel):
|
|||||||
do_dropout = random.random() < self.inpaint_dropout if self.inpaint_dropout > 0.0 else False
|
do_dropout = random.random() < self.inpaint_dropout if self.inpaint_dropout > 0.0 else False
|
||||||
# do random mask if we dont have one
|
# do random mask if we dont have one
|
||||||
inpaint_tensor = batch.inpaint_tensor
|
inpaint_tensor = batch.inpaint_tensor
|
||||||
|
if inpaint_tensor is None and batch.mask_tensor is not None:
|
||||||
|
# we have a mask tensor, use it
|
||||||
|
inpaint_tensor = batch.mask_tensor
|
||||||
|
|
||||||
if self.inpaint_random_chance > 0.0:
|
if self.inpaint_random_chance > 0.0:
|
||||||
do_random = random.random() < self.inpaint_random_chance
|
do_random = random.random() < self.inpaint_random_chance
|
||||||
if do_random:
|
if do_random:
|
||||||
# force a random tensor
|
# force a random tensor
|
||||||
inpaint_tensor = None
|
inpaint_tensor = None
|
||||||
|
|
||||||
if inpaint_tensor is None and not do_dropout:
|
if inpaint_tensor is None and not do_dropout and self.do_random_inpainting:
|
||||||
# generate a random one since we dont have one
|
# generate a random one since we dont have one
|
||||||
# this will make random blobs, invert the blobs for now as we normanlly inpaint the alpha
|
# this will make random blobs, invert the blobs for now as we normanlly inpaint the alpha
|
||||||
inpaint_tensor = 1 - generate_random_mask(
|
inpaint_tensor = 1 - generate_random_mask(
|
||||||
@@ -393,16 +409,32 @@ class Flex2(BaseModel):
|
|||||||
elif inpaint_tensor.shape[1] == 3:
|
elif inpaint_tensor.shape[1] == 3:
|
||||||
# rgb mask. Just get one channel
|
# rgb mask. Just get one channel
|
||||||
inpainting_tensor_mask = inpaint_tensor[:, 0:1, :, :].to(latents.device, dtype=latents.dtype)
|
inpainting_tensor_mask = inpaint_tensor[:, 0:1, :, :].to(latents.device, dtype=latents.dtype)
|
||||||
|
# mask is 0-1 with 1 being inpaint area, we need to invert it for now, it is re inverted later
|
||||||
|
inpaint_tensor = 1 - inpaint_tensor
|
||||||
else:
|
else:
|
||||||
inpainting_tensor_mask = inpaint_tensor
|
inpainting_tensor_mask = inpaint_tensor
|
||||||
|
|
||||||
# # use our batch latents so we cna avoid ancoding again
|
# # use our batch latents so we cna avoid encoding again
|
||||||
inpainting_latent = batch.latents
|
inpainting_latent = batch.latents
|
||||||
|
|
||||||
# resize the mask to match the new encoded size
|
# resize the mask to match the new encoded size
|
||||||
inpainting_tensor_mask = F.interpolate(inpainting_tensor_mask, size=(inpainting_latent.shape[2], inpainting_latent.shape[3]), mode='bilinear')
|
inpainting_tensor_mask = F.interpolate(inpainting_tensor_mask, size=(inpainting_latent.shape[2], inpainting_latent.shape[3]), mode='bilinear')
|
||||||
inpainting_tensor_mask = inpainting_tensor_mask.to(latents.device, latents.dtype)
|
inpainting_tensor_mask = inpainting_tensor_mask.to(latents.device, latents.dtype)
|
||||||
|
|
||||||
|
if self.random_blur_mask:
|
||||||
|
# blur the mask
|
||||||
|
# Give it a channel dim of 1
|
||||||
|
inpainting_tensor_mask = inpainting_tensor_mask.unsqueeze(1)
|
||||||
|
# we are at latent size, so keep kernel smaller
|
||||||
|
inpainting_tensor_mask = random_blur(
|
||||||
|
inpainting_tensor_mask,
|
||||||
|
min_kernel_size=3,
|
||||||
|
max_kernel_size=8,
|
||||||
|
p=0.5
|
||||||
|
)
|
||||||
|
# remove the channel dim
|
||||||
|
inpainting_tensor_mask = inpainting_tensor_mask.squeeze(1)
|
||||||
|
|
||||||
do_mask_invert = False
|
do_mask_invert = False
|
||||||
if self.invert_inpaint_mask_chance > 0.0:
|
if self.invert_inpaint_mask_chance > 0.0:
|
||||||
do_mask_invert = random.random() < self.invert_inpaint_mask_chance
|
do_mask_invert = random.random() < self.invert_inpaint_mask_chance
|
||||||
|
|||||||
Reference in new issue
Block a user