Add Qwen3.6 VL captioner.

This commit is contained in:
Jaret Burkett
2026-07-15 07:02:33 -06:00
parent 18da85153b
commit 691ddf434e
5 changed files with 68 additions and 13 deletions

View File

@@ -40,6 +40,7 @@ class CaptionConfig:
self.recaption = kwargs.get("recaption", False) self.recaption = kwargs.get("recaption", False)
self.max_res = kwargs.get("max_res", 512) self.max_res = kwargs.get("max_res", 512)
self.max_new_tokens = kwargs.get("max_new_tokens", 128) self.max_new_tokens = kwargs.get("max_new_tokens", 128)
self.thinking = kwargs.get("thinking", False)
self.caption_prompt = kwargs.get( self.caption_prompt = kwargs.get(
"caption_prompt", "Describe this image in detail." "caption_prompt", "Describe this image in detail."
) )

View File

@@ -1,7 +1,8 @@
from transformers import ( from transformers import (
Qwen3VLForConditionalGeneration, AutoModelForImageTextToText,
Qwen3VLMoeForConditionalGeneration,
AutoProcessor, AutoProcessor,
StoppingCriteria,
StoppingCriteriaList,
) )
from collections import OrderedDict from collections import OrderedDict
@@ -44,6 +45,27 @@ def patch_qwen_vl_patch_embed(model):
warnings.filterwarnings("ignore") warnings.filterwarnings("ignore")
logging.disable(logging.WARNING) logging.disable(logging.WARNING)
# hard cap on reasoning tokens so a runaway think block cannot generate forever
MAX_THINKING_TOKENS = 4096
class ThinkingBudgetCriteria(StoppingCriteria):
"""For thinking models: lets the model reason freely, then counts
max_new_tokens starting from the token after </think> so the visible answer
gets the full budget regardless of how long the reasoning ran."""
def __init__(self, think_end_token_id: int, max_new_tokens: int):
self.think_end_token_id = think_end_token_id
self.max_new_tokens = max_new_tokens
self.answer_start = None
def __call__(self, input_ids, scores, **kwargs):
if self.answer_start is None:
if input_ids[0, -1].item() == self.think_end_token_id:
self.answer_start = input_ids.shape[1]
return False
return (input_ids.shape[1] - self.answer_start) >= self.max_new_tokens
class Qwen3VLCaptioner(BaseCaptioner): class Qwen3VLCaptioner(BaseCaptioner):
def __init__(self, process_id: int, job, config: OrderedDict, **kwargs): def __init__(self, process_id: int, job, config: OrderedDict, **kwargs):
@@ -51,12 +73,7 @@ class Qwen3VLCaptioner(BaseCaptioner):
def load_model(self): def load_model(self):
self.print_and_status_update("Loading Qwen3VL model") self.print_and_status_update("Loading Qwen3VL model")
ModelClass = ( self.model = AutoModelForImageTextToText.from_pretrained(
Qwen3VLMoeForConditionalGeneration
if "B-A" in self.caption_config.model_name_or_path
else Qwen3VLForConditionalGeneration
)
self.model = ModelClass.from_pretrained(
self.caption_config.model_name_or_path, self.caption_config.model_name_or_path,
dtype=self.torch_dtype, dtype=self.torch_dtype,
device_map="cpu", device_map="cpu",
@@ -100,13 +117,33 @@ class Qwen3VLCaptioner(BaseCaptioner):
add_generation_prompt=True, add_generation_prompt=True,
return_dict=True, return_dict=True,
return_tensors="pt", return_tensors="pt",
enable_thinking=self.caption_config.thinking,
) )
inputs = inputs.to(self.device_torch) inputs = inputs.to(self.device_torch)
gen_kwargs = {"max_new_tokens": self.caption_config.max_new_tokens}
if self.caption_config.thinking:
think_end_token_id = self.processor.tokenizer.convert_tokens_to_ids(
"</think>"
)
if think_end_token_id is not None:
# give the model room to think, but start the max_new_tokens
# budget only once the think block closes
gen_kwargs = {
"max_new_tokens": MAX_THINKING_TOKENS
+ self.caption_config.max_new_tokens,
"stopping_criteria": StoppingCriteriaList(
[
ThinkingBudgetCriteria(
think_end_token_id,
self.caption_config.max_new_tokens,
)
]
),
}
# Inference: Generation of the output # Inference: Generation of the output
generated_ids = self.model.generate( generated_ids = self.model.generate(**inputs, **gen_kwargs)
**inputs, max_new_tokens=self.caption_config.max_new_tokens
)
generated_ids_trimmed = [ generated_ids_trimmed = [
out_ids[len(in_ids) :] out_ids[len(in_ids) :]
for in_ids, out_ids in zip(inputs.input_ids, generated_ids) for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
@@ -117,7 +154,12 @@ class Qwen3VLCaptioner(BaseCaptioner):
clean_up_tokenization_spaces=False, clean_up_tokenization_spaces=False,
) )
return output_text[0].strip() caption = output_text[0]
# thinking models (e.g. Qwen3.6) may still emit reasoning before the
# answer; keep only what follows the think block
if "</think>" in caption:
caption = caption.split("</think>")[-1]
return caption.strip()
except Exception as e: except Exception as e:
print(f"Error processing {file_path}: {e}") print(f"Error processing {file_path}: {e}")
traceback.print_exc() traceback.print_exc()

View File

@@ -183,6 +183,13 @@ const CaptionSimpleJob: React.FC<Props> = ({ jobConfig, setJobConfig, gpuIDs, se
checked={jobConfig.config.process[0].caption.compile || false} checked={jobConfig.config.process[0].caption.compile || false}
onChange={value => setJobConfig(value, 'config.process[0].caption.compile')} onChange={value => setJobConfig(value, 'config.process[0].caption.compile')}
/> />
{additionalSections.includes('caption.thinking') && (
<Checkbox
label="Thinking"
checked={jobConfig.config.process[0].caption.thinking || false}
onChange={value => setJobConfig(value, 'config.process[0].caption.thinking')}
/>
)}
</FormGroup> </FormGroup>
</div> </div>
</div> </div>

View File

@@ -1,7 +1,7 @@
import { GroupedSelectOption, SelectOption } from "@/types"; import { GroupedSelectOption, SelectOption } from "@/types";
type CaptionGroup = 'image' | 'music'; type CaptionGroup = 'image' | 'music';
type AdditionalSections = 'caption.model_name_or_path2' | 'caption.caption_prompt' | 'caption.max_res' | 'caption.max_new_tokens' | 'caption.fixed_caption'; type AdditionalSections = 'caption.model_name_or_path2' | 'caption.caption_prompt' | 'caption.max_res' | 'caption.max_new_tokens' | 'caption.fixed_caption' | 'caption.thinking';
export interface CaptionOption { export interface CaptionOption {
name: string; name: string;
@@ -66,12 +66,16 @@ export const captionerTypes: CaptionOption[] = [
{ value: 'Qwen/Qwen3-VL-2B-Instruct', label: 'Qwen/Qwen3-VL-2B-Instruct' }, { value: 'Qwen/Qwen3-VL-2B-Instruct', label: 'Qwen/Qwen3-VL-2B-Instruct' },
{ value: 'Qwen/Qwen3-VL-4B-Instruct', label: 'Qwen/Qwen3-VL-4B-Instruct' }, { value: 'Qwen/Qwen3-VL-4B-Instruct', label: 'Qwen/Qwen3-VL-4B-Instruct' },
{ value: 'Qwen/Qwen3-VL-8B-Instruct', label: 'Qwen/Qwen3-VL-8B-Instruct' }, { value: 'Qwen/Qwen3-VL-8B-Instruct', label: 'Qwen/Qwen3-VL-8B-Instruct' },
{ value: 'huihui-ai/Huihui-Qwen3-VL-8B-Instruct-abliterated', label: 'huihui-ai/Huihui-Qwen3-VL-8B-Instruct-abliterated' },
{ value: 'Qwen/Qwen3-VL-30B-A3B-Instruct', label: 'Qwen/Qwen3-VL-30B-A3B-Instruct' }, { value: 'Qwen/Qwen3-VL-30B-A3B-Instruct', label: 'Qwen/Qwen3-VL-30B-A3B-Instruct' },
{ value: 'Qwen/Qwen3.6-27B', label: 'Qwen/Qwen3.6-27B' },
{ value: 'huihui-ai/Huihui-Qwen3.6-27B-abliterated', label: 'huihui-ai/Huihui-Qwen3.6-27B-abliterated' },
], ],
additionalSections: [ additionalSections: [
'caption.caption_prompt', 'caption.caption_prompt',
'caption.max_res', 'caption.max_res',
'caption.max_new_tokens', 'caption.max_new_tokens',
'caption.thinking',
], ],
}, },
{ {

View File

@@ -281,6 +281,7 @@ export interface CaptionProcessConfig {
max_new_tokens?: number; max_new_tokens?: number;
fixed_caption?: string; fixed_caption?: string;
caption_extension?: string; caption_extension?: string;
thinking?: boolean;
} }
} }