Added some additional experimental things to the vision direct encoder

toolkit/dataloader_mixins.py

@@ -13,7 +13,7 @@ import numpy as np
 import torch
 from safetensors.torch import load_file, save_file
 from tqdm import tqdm
-from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection
+from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection, SiglipImageProcessor
 
 from toolkit.basic import flush, value_map
 from toolkit.buckets import get_bucket_for_image_size, get_resolution
@@ -764,7 +764,8 @@ class ClipImageFileItemDTOMixin:
             return self.clip_image_path
 
     def load_clip_image(self: 'FileItemDTO'):
-        is_dynamic_size_and_aspect = isinstance(self.clip_image_processor, PixtralVisionImagePreprocessorCompatible)
+        is_dynamic_size_and_aspect = isinstance(self.clip_image_processor, PixtralVisionImagePreprocessorCompatible) or \
+                                    isinstance(self.clip_image_processor, SiglipImageProcessor)
         if self.is_vision_clip_cached:
             self.clip_image_embeds = load_file(self.get_clip_vision_embeddings_path())
 
@@ -794,21 +795,7 @@ class ClipImageFileItemDTOMixin:
             img = img.transpose(Image.FLIP_TOP_BOTTOM)
             
         if is_dynamic_size_and_aspect:
-            # just match the bucket size for now
-            if self.dataset_config.buckets:
-                # scale and crop based on file item
-                img = img.resize((self.scale_to_width, self.scale_to_height), Image.BICUBIC)
-                # img = transforms.CenterCrop((self.crop_height, self.crop_width))(img)
-                # crop
-                img = img.crop((
-                    self.crop_x,
-                    self.crop_y,
-                    self.crop_x + self.crop_width,
-                    self.crop_y + self.crop_height
-                ))
-            else:
-                raise Exception("Control images not supported for non-bucket datasets")
-        
+            pass  # let the image processor handle it
         elif img.width != img.height:
             min_size = min(img.width, img.height)
             if self.dataset_config.square_crop: