Added ability to load clip pairs randomly from folder. Other small bug fixes

This commit is contained in:
Jaret Burkett committed 2024-10-03 10:03:49 -06:00
1 parent f05224970f
commit 67e0aca750
4 files changed
+57 -28

No files matched your search

+24 -22
View File
@@ -34,6 +34,7 @@ class GenerateConfig:
self.compile = kwargs.get('compile', False) self.compile = kwargs.get('compile', False)
self.ext = kwargs.get('ext', 'png') self.ext = kwargs.get('ext', 'png')
self.prompt_file = kwargs.get('prompt_file', False) self.prompt_file = kwargs.get('prompt_file', False)
self.num_repeats = kwargs.get('num_repeats', 1)
self.prompts_in_file = self.prompts self.prompts_in_file = self.prompts
if self.prompts is None: if self.prompts is None:
raise ValueError("Prompts must be set") raise ValueError("Prompts must be set")
@@ -110,30 +111,31 @@ class GenerateProcess(BaseProcess):
print(f"Generating {len(self.generate_config.prompts)} images") print(f"Generating {len(self.generate_config.prompts)} images")
# build prompt image configs # build prompt image configs
prompt_image_configs = [] prompt_image_configs = []
for prompt in self.generate_config.prompts: for _ in range(self.generate_config.num_repeats):
width = self.generate_config.width for prompt in self.generate_config.prompts:
height = self.generate_config.height width = self.generate_config.width
prompt = self.clean_prompt(prompt) height = self.generate_config.height
# prompt = self.clean_prompt(prompt)
if self.generate_config.size_list is not None: if self.generate_config.size_list is not None:
# randomly select a size # randomly select a size
width, height = random.choice(self.generate_config.size_list) width, height = random.choice(self.generate_config.size_list)
prompt_image_configs.append(GenerateImageConfig( prompt_image_configs.append(GenerateImageConfig(
prompt=prompt, prompt=prompt,
prompt_2=self.generate_config.prompt_2, prompt_2=self.generate_config.prompt_2,
width=width, width=width,
height=height, height=height,
num_inference_steps=self.generate_config.sample_steps, num_inference_steps=self.generate_config.sample_steps,
guidance_scale=self.generate_config.guidance_scale, guidance_scale=self.generate_config.guidance_scale,
negative_prompt=self.generate_config.neg, negative_prompt=self.generate_config.neg,
negative_prompt_2=self.generate_config.neg_2, negative_prompt_2=self.generate_config.neg_2,
seed=self.generate_config.seed, seed=self.generate_config.seed,
guidance_rescale=self.generate_config.guidance_rescale, guidance_rescale=self.generate_config.guidance_rescale,
output_ext=self.generate_config.ext, output_ext=self.generate_config.ext,
output_folder=self.output_folder, output_folder=self.output_folder,
add_prompt_file=self.generate_config.prompt_file add_prompt_file=self.generate_config.prompt_file
)) ))
# generate images # generate images
self.sd.generate_images(prompt_image_configs, sampler=self.generate_config.sampler) self.sd.generate_images(prompt_image_configs, sampler=self.generate_config.sampler)
+2
View File
@@ -607,6 +607,8 @@ class DatasetConfig:
# ip adapter / reference dataset # ip adapter / reference dataset
self.clip_image_path: str = kwargs.get('clip_image_path', None) # depth maps, etc self.clip_image_path: str = kwargs.get('clip_image_path', None) # depth maps, etc
# get the clip image randomly from the same folder as the image. Useful for folder grouped pairs.
self.clip_image_from_same_folder: bool = kwargs.get('clip_image_from_same_folder', False)
self.clip_image_augmentations: List[dict] = kwargs.get('clip_image_augmentations', None) self.clip_image_augmentations: List[dict] = kwargs.get('clip_image_augmentations', None)
self.clip_image_shuffle_augmentations: bool = kwargs.get('clip_image_shuffle_augmentations', False) self.clip_image_shuffle_augmentations: bool = kwargs.get('clip_image_shuffle_augmentations', False)
self.replacements: List[str] = kwargs.get('replacements', []) self.replacements: List[str] = kwargs.get('replacements', [])
+29 -4
View File
@@ -1,4 +1,5 @@
import base64 import base64
import glob
import hashlib import hashlib
import json import json
import math import math
@@ -630,11 +631,12 @@ class ClipImageFileItemDTOMixin:
self.clip_vision_unconditional_paths: Union[List[str], None] = None self.clip_vision_unconditional_paths: Union[List[str], None] = None
self._clip_vision_embeddings_path: Union[str, None] = None self._clip_vision_embeddings_path: Union[str, None] = None
dataset_config: 'DatasetConfig' = kwargs.get('dataset_config', None) dataset_config: 'DatasetConfig' = kwargs.get('dataset_config', None)
if dataset_config.clip_image_path is not None: if dataset_config.clip_image_path is not None or dataset_config.clip_image_from_same_folder:
# copy the clip image processor so the dataloader can do it # copy the clip image processor so the dataloader can do it
sd = kwargs.get('sd', None) sd = kwargs.get('sd', None)
if hasattr(sd.adapter, 'clip_image_processor'): if hasattr(sd.adapter, 'clip_image_processor'):
self.clip_image_processor = sd.adapter.clip_image_processor self.clip_image_processor = sd.adapter.clip_image_processor
if dataset_config.clip_image_path is not None:
# find the control image path # find the control image path
clip_image_path = dataset_config.clip_image_path clip_image_path = dataset_config.clip_image_path
# we are using control images # we are using control images
@@ -646,7 +648,11 @@ class ClipImageFileItemDTOMixin:
self.clip_image_path = os.path.join(clip_image_path, file_name_no_ext + ext) self.clip_image_path = os.path.join(clip_image_path, file_name_no_ext + ext)
self.has_clip_image = True self.has_clip_image = True
break break
self.build_clip_imag_augmentation_transform()
if dataset_config.clip_image_from_same_folder:
# assume we have one. We will pull it on load.
self.has_clip_image = True
self.build_clip_imag_augmentation_transform() self.build_clip_imag_augmentation_transform()
def build_clip_imag_augmentation_transform(self: 'FileItemDTO'): def build_clip_imag_augmentation_transform(self: 'FileItemDTO'):
@@ -732,6 +738,24 @@ class ClipImageFileItemDTOMixin:
self._clip_vision_embeddings_path = os.path.join(latent_dir, f'{filename_no_ext}_{hash_str}.safetensors') self._clip_vision_embeddings_path = os.path.join(latent_dir, f'{filename_no_ext}_{hash_str}.safetensors')
return self._clip_vision_embeddings_path return self._clip_vision_embeddings_path
def get_new_clip_image_path(self: 'FileItemDTO'):
if self.dataset_config.clip_image_from_same_folder:
# randomly grab an image path from the same folder
pool_folder = os.path.dirname(self.path)
# find all images in the folder
img_ext_list = ['.jpg', '.jpeg', '.png', '.webp']
img_files = []
for ext in img_ext_list:
img_files += glob.glob(os.path.join(pool_folder, f'*{ext}'))
# remove the current image if len is greater than 1
if len(img_files) > 1:
img_files.remove(self.path)
# randomly grab one
self.clip_image_path = random.choice(img_files)
return self.path
else:
return self.clip_image_path
def load_clip_image(self: 'FileItemDTO'): def load_clip_image(self: 'FileItemDTO'):
is_dynamic_size_and_aspect = isinstance(self.clip_image_processor, PixtralVisionImagePreprocessorCompatible) is_dynamic_size_and_aspect = isinstance(self.clip_image_processor, PixtralVisionImagePreprocessorCompatible)
@@ -744,14 +768,15 @@ class ClipImageFileItemDTOMixin:
self.clip_image_embeds_unconditional = load_file(unconditional_path) self.clip_image_embeds_unconditional = load_file(unconditional_path)
return return
clip_image_path = self.get_new_clip_image_path()
try: try:
img = Image.open(self.clip_image_path).convert('RGB') img = Image.open(clip_image_path).convert('RGB')
img = exif_transpose(img) img = exif_transpose(img)
except Exception as e: except Exception as e:
# make a random noise image # make a random noise image
img = Image.new('RGB', (self.dataset_config.resolution, self.dataset_config.resolution)) img = Image.new('RGB', (self.dataset_config.resolution, self.dataset_config.resolution))
print(f"Error: {e}") print(f"Error: {e}")
print(f"Error loading image: {self.clip_image_path}") print(f"Error loading image: {clip_image_path}")
img = img.convert('RGB') img = img.convert('RGB')
+2 -2
View File
@@ -472,8 +472,8 @@ class VisionDirectAdapter(torch.nn.Module):
self.mid_size = self.token_size self.mid_size = self.token_size
# if pixtral, use cross attn dim for more sparse representation # if pixtral, use cross attn dim for more sparse representation if only doing double transformers
if is_pixtral: if is_pixtral and self.config.flux_only_double:
if is_flux: if is_flux:
hidden_size = 3072 hidden_size = 3072
else: else: