Add layer offloading to omni captioner

ui/src/components/CaptionSimpleJob.tsx

@@ -4,6 +4,7 @@ import {
   CreatableSelectInput,
   FormGroup,
   SelectInput,
+  SliderInput,
   TextAreaInput,
   TextInput,
 } from '@/components/formInputs';
@@ -208,6 +209,29 @@ const CaptionSimpleJob: React.FC<Props> = ({ jobConfig, setJobConfig, gpuIDs, se
                 onChange={value => setJobConfig(value, 'config.process[0].caption.thinking')}
               />
             )}
+            {additionalSections.includes('caption.layer_offloading') && (
+              <>
+                <Checkbox
+                  label="Layer Offloading"
+                  checked={jobConfig.config.process[0].caption.layer_offloading || false}
+                  onChange={value => setJobConfig(value, 'config.process[0].caption.layer_offloading')}
+                />
+                {jobConfig.config.process[0].caption.layer_offloading && (
+                  <div className="pt-2">
+                    <SliderInput
+                      label="Offload %"
+                      value={Math.round((jobConfig.config.process[0].caption.layer_offloading_percent ?? 1) * 100)}
+                      onChange={value =>
+                        setJobConfig(value * 0.01, 'config.process[0].caption.layer_offloading_percent')
+                      }
+                      min={0}
+                      max={100}
+                      step={1}
+                    />
+                  </div>
+                )}
+              </>
+            )}
           </FormGroup>
         </div>
       </div>

ui/src/helpers/captionOptions.ts

@@ -1,7 +1,7 @@
 import { GroupedSelectOption, SelectOption } from "@/types";
 
 type CaptionGroup = 'image' | 'music' | 'video';
-type AdditionalSections = 'caption.model_name_or_path2' | 'caption.caption_prompt' | 'caption.max_res' | 'caption.max_new_tokens' | 'caption.fixed_caption' | 'caption.thinking' | 'caption.batch_size';
+type AdditionalSections = 'caption.model_name_or_path2' | 'caption.caption_prompt' | 'caption.max_res' | 'caption.max_new_tokens' | 'caption.fixed_caption' | 'caption.thinking' | 'caption.batch_size' | 'caption.layer_offloading';
 
 export interface CaptionOption {
     name: string;
@@ -130,6 +130,7 @@ export const captionerTypes: CaptionOption[] = [
             'caption.max_res',
             'caption.max_new_tokens',
             'caption.batch_size',
+            'caption.layer_offloading',
         ],
     },
     {
@@ -179,8 +180,8 @@ export const groupedCaptionerTypes: GroupedSelectOption[] = captionerTypes.reduc
 
 export const quantizationOptions: SelectOption[] = [
     { value: '', label: '- NONE -' },
-    { value: 'float8', label: 'float8 (default)' },
-    { value: 'convrot8', label: '8bit convrot' },
+    { value: 'float8', label: 'float8' },
+    { value: 'convrot8', label: '8bit convrot (default)' },
     { value: 'convrot4', label: '4bit convrot (nvfp4)' },
     { value: 'convrotint7', label: '7bit convrot' },
     { value: 'convrotint6', label: '6bit convrot' },

ui/src/types.ts

@@ -299,6 +299,8 @@ export interface CaptionProcessConfig {
     caption_extension?: string;
     thinking?: boolean;
     batch_size?: number;
+    layer_offloading?: boolean;
+    layer_offloading_percent?: number;
   }
 }