github/ai-toolkit
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Added blank prompt preservation

Browse Source
This commit is contained in:
Jaret Burkett
2025-10-22 14:55:13 -06:00
parent ca57ffc270
commit ee206cfa18
7 changed files with 143 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
ui/src/app/jobs/new/SimpleJob.tsx
View File

@@ -215,12 +215,12 @@ export default function SimpleJob({
</FormGroup>
)}
{modelArch?.additionalSections?.includes('model.qie.match_target_res') && (
<Checkbox
label="Match Target Res"
docKey="model.qie.match_target_res"
checked={jobConfig.config.process[0].model.model_kwargs.match_target_res}
onChange={value => setJobConfig(value, 'config.process[0].model.model_kwargs.match_target_res')}
/>
<Checkbox
label="Match Target Res"
docKey="model.qie.match_target_res"
checked={jobConfig.config.process[0].model.model_kwargs.match_target_res}
onChange={value => setJobConfig(value, 'config.process[0].model.model_kwargs.match_target_res')}
/>
)}
{modelArch?.additionalSections?.includes('model.layer_offloading') && (
<>
@@ -586,16 +586,27 @@ export default function SimpleJob({
</FormGroup>
</div>
<div>
{disableSections.includes('train.diff_output_preservation') ||
disableSections.includes('train.blank_prompt_preservation') ? null : (
<FormGroup label="Regularization">
<></>
</FormGroup>
)}
{disableSections.includes('train.diff_output_preservation') ? null : (
<>
<FormGroup label="Regularization">
<Checkbox
label="Differential Output Preservation"
className="pt-1"
checked={jobConfig.config.process[0].train.diff_output_preservation || false}
onChange={value => setJobConfig(value, 'config.process[0].train.diff_output_preservation')}
/>
</FormGroup>
<Checkbox
label="Differential Output Preservation"
docKey={'train.diff_output_preservation'}
className="pt-1"
checked={jobConfig.config.process[0].train.diff_output_preservation || false}
onChange={value => {
setJobConfig(value, 'config.process[0].train.diff_output_preservation');
if (value && jobConfig.config.process[0].train.blank_prompt_preservation) {
// only one can be enabled at a time
setJobConfig(false, 'config.process[0].train.blank_prompt_preservation');
}
}}
/>
{jobConfig.config.process[0].train.diff_output_preservation && (
<>
<NumberInput
@@ -610,7 +621,7 @@ export default function SimpleJob({
/>
<TextInput
label="DOP Preservation Class"
className="pt-2"
className="pt-2 pb-4"
value={jobConfig.config.process[0].train.diff_output_preservation_class as string}
onChange={value =>
setJobConfig(value, 'config.process[0].train.diff_output_preservation_class')
@@ -621,6 +632,39 @@ export default function SimpleJob({
)}
</>
)}
{disableSections.includes('train.blank_prompt_preservation') ? null : (
<>
<Checkbox
label="Blank Prompt Preservation"
docKey={'train.blank_prompt_preservation'}
className="pt-1"
checked={jobConfig.config.process[0].train.blank_prompt_preservation || false}
onChange={value => {
setJobConfig(value, 'config.process[0].train.blank_prompt_preservation');
if (value && jobConfig.config.process[0].train.diff_output_preservation) {
// only one can be enabled at a time
setJobConfig(false, 'config.process[0].train.diff_output_preservation');
}
}}
/>
{jobConfig.config.process[0].train.blank_prompt_preservation && (
<>
<NumberInput
label="BPP Loss Multiplier"
className="pt-2"
value={
(jobConfig.config.process[0].train.blank_prompt_preservation_multiplier as number) || 1.0
}
onChange={value =>
setJobConfig(value, 'config.process[0].train.blank_prompt_preservation_multiplier')
}
placeholder="eg. 1.0"
min={0}
/>
</>
)}
</>
)}
</div>
</div>
</Card>

1
ui/src/app/jobs/new/options.ts
View File

@@ -9,6 +9,7 @@ type DisableableSections =
| 'network.conv'
| 'trigger_word'
| 'train.diff_output_preservation'
| 'train.blank_prompt_preservation'
| 'train.unload_text_encoder'
| 'slider';

30
ui/src/docs.tsx
View File

@@ -228,6 +228,36 @@ const docs: { [key: string]: ConfigDoc } = {
</>
),
},
'train.diff_output_preservation': {
title: 'Differential Output Preservation',
description: (
<>
Differential Output Preservation (DOP) is a technique to help preserve class of the trained concept during
training. For this, you must have a trigger word set to differentiate your concept from its class. For instance,
You may be training a woman named Alice. Your trigger word may be "Alice". The class is "woman", since Alice is
a woman. We want to teach the model to remember what it knows about the class "woman" while teaching it what is
different about Alice. During training, the trainer will make a prediction with your LoRA bypassed and your
trigger word in the prompt replaced with the class word. Making "photo of Alice" become "photo of woman". This
prediction is called the prior prediction. Each step, we will do the normal training step, but also do another
step with this prior prediction and the class prompt in order to teach our LoRA to preserve the knowledge of the
class. This should not only improve the performance of your trained concept, but also allow you to do things
like "Alice standing next to a woman" and not make both of the people look like Alice.
</>
),
},
'train.blank_prompt_preservation': {
title: 'Blank Prompt Preservation',
description: (
<>
Blank Prompt Preservation (BPP) is a technique to help preserve the current models knowledge when unprompted.
This will not only help the model become more flexible, but will also help the quality of your concept during
inference, especially when a model uses CFG (Classifier Free Guidance) on inference. At each step during
training, a prior prediction is made with a blank prompt and with the LoRA disabled. This prediction is then
used as a target on an additional training step with a blank prompt, to preserve the model's knowledge when no
prompt is given. This helps the model to not overfit to the prompt and retain its generalization capabilities.
</>
),
},
};
export const getDoc = (key: string | null | undefined): ConfigDoc | null => {

2
ui/src/types.ts
View File

@@ -135,6 +135,8 @@ export interface TrainConfig {
diff_output_preservation: boolean;
diff_output_preservation_multiplier: number;
diff_output_preservation_class: string;
blank_prompt_preservation?: boolean;
blank_prompt_preservation_multiplier?: number;
switch_boundary_every: number;
loss_type: 'mse' | 'mae' | 'wavelet' | 'stepped';
}