Add Qwen3.6 VL captioner.
This commit is contained in:
@@ -40,6 +40,7 @@ class CaptionConfig:
|
|||||||
self.recaption = kwargs.get("recaption", False)
|
self.recaption = kwargs.get("recaption", False)
|
||||||
self.max_res = kwargs.get("max_res", 512)
|
self.max_res = kwargs.get("max_res", 512)
|
||||||
self.max_new_tokens = kwargs.get("max_new_tokens", 128)
|
self.max_new_tokens = kwargs.get("max_new_tokens", 128)
|
||||||
|
self.thinking = kwargs.get("thinking", False)
|
||||||
self.caption_prompt = kwargs.get(
|
self.caption_prompt = kwargs.get(
|
||||||
"caption_prompt", "Describe this image in detail."
|
"caption_prompt", "Describe this image in detail."
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
from transformers import (
|
from transformers import (
|
||||||
Qwen3VLForConditionalGeneration,
|
AutoModelForImageTextToText,
|
||||||
Qwen3VLMoeForConditionalGeneration,
|
|
||||||
AutoProcessor,
|
AutoProcessor,
|
||||||
|
StoppingCriteria,
|
||||||
|
StoppingCriteriaList,
|
||||||
)
|
)
|
||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
|
|
||||||
@@ -44,6 +45,27 @@ def patch_qwen_vl_patch_embed(model):
|
|||||||
warnings.filterwarnings("ignore")
|
warnings.filterwarnings("ignore")
|
||||||
logging.disable(logging.WARNING)
|
logging.disable(logging.WARNING)
|
||||||
|
|
||||||
|
# hard cap on reasoning tokens so a runaway think block cannot generate forever
|
||||||
|
MAX_THINKING_TOKENS = 4096
|
||||||
|
|
||||||
|
|
||||||
|
class ThinkingBudgetCriteria(StoppingCriteria):
|
||||||
|
"""For thinking models: lets the model reason freely, then counts
|
||||||
|
max_new_tokens starting from the token after </think> so the visible answer
|
||||||
|
gets the full budget regardless of how long the reasoning ran."""
|
||||||
|
|
||||||
|
def __init__(self, think_end_token_id: int, max_new_tokens: int):
|
||||||
|
self.think_end_token_id = think_end_token_id
|
||||||
|
self.max_new_tokens = max_new_tokens
|
||||||
|
self.answer_start = None
|
||||||
|
|
||||||
|
def __call__(self, input_ids, scores, **kwargs):
|
||||||
|
if self.answer_start is None:
|
||||||
|
if input_ids[0, -1].item() == self.think_end_token_id:
|
||||||
|
self.answer_start = input_ids.shape[1]
|
||||||
|
return False
|
||||||
|
return (input_ids.shape[1] - self.answer_start) >= self.max_new_tokens
|
||||||
|
|
||||||
|
|
||||||
class Qwen3VLCaptioner(BaseCaptioner):
|
class Qwen3VLCaptioner(BaseCaptioner):
|
||||||
def __init__(self, process_id: int, job, config: OrderedDict, **kwargs):
|
def __init__(self, process_id: int, job, config: OrderedDict, **kwargs):
|
||||||
@@ -51,12 +73,7 @@ class Qwen3VLCaptioner(BaseCaptioner):
|
|||||||
|
|
||||||
def load_model(self):
|
def load_model(self):
|
||||||
self.print_and_status_update("Loading Qwen3VL model")
|
self.print_and_status_update("Loading Qwen3VL model")
|
||||||
ModelClass = (
|
self.model = AutoModelForImageTextToText.from_pretrained(
|
||||||
Qwen3VLMoeForConditionalGeneration
|
|
||||||
if "B-A" in self.caption_config.model_name_or_path
|
|
||||||
else Qwen3VLForConditionalGeneration
|
|
||||||
)
|
|
||||||
self.model = ModelClass.from_pretrained(
|
|
||||||
self.caption_config.model_name_or_path,
|
self.caption_config.model_name_or_path,
|
||||||
dtype=self.torch_dtype,
|
dtype=self.torch_dtype,
|
||||||
device_map="cpu",
|
device_map="cpu",
|
||||||
@@ -100,13 +117,33 @@ class Qwen3VLCaptioner(BaseCaptioner):
|
|||||||
add_generation_prompt=True,
|
add_generation_prompt=True,
|
||||||
return_dict=True,
|
return_dict=True,
|
||||||
return_tensors="pt",
|
return_tensors="pt",
|
||||||
|
enable_thinking=self.caption_config.thinking,
|
||||||
)
|
)
|
||||||
inputs = inputs.to(self.device_torch)
|
inputs = inputs.to(self.device_torch)
|
||||||
|
|
||||||
|
gen_kwargs = {"max_new_tokens": self.caption_config.max_new_tokens}
|
||||||
|
if self.caption_config.thinking:
|
||||||
|
think_end_token_id = self.processor.tokenizer.convert_tokens_to_ids(
|
||||||
|
"</think>"
|
||||||
|
)
|
||||||
|
if think_end_token_id is not None:
|
||||||
|
# give the model room to think, but start the max_new_tokens
|
||||||
|
# budget only once the think block closes
|
||||||
|
gen_kwargs = {
|
||||||
|
"max_new_tokens": MAX_THINKING_TOKENS
|
||||||
|
+ self.caption_config.max_new_tokens,
|
||||||
|
"stopping_criteria": StoppingCriteriaList(
|
||||||
|
[
|
||||||
|
ThinkingBudgetCriteria(
|
||||||
|
think_end_token_id,
|
||||||
|
self.caption_config.max_new_tokens,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
# Inference: Generation of the output
|
# Inference: Generation of the output
|
||||||
generated_ids = self.model.generate(
|
generated_ids = self.model.generate(**inputs, **gen_kwargs)
|
||||||
**inputs, max_new_tokens=self.caption_config.max_new_tokens
|
|
||||||
)
|
|
||||||
generated_ids_trimmed = [
|
generated_ids_trimmed = [
|
||||||
out_ids[len(in_ids) :]
|
out_ids[len(in_ids) :]
|
||||||
for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
|
for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
|
||||||
@@ -117,7 +154,12 @@ class Qwen3VLCaptioner(BaseCaptioner):
|
|||||||
clean_up_tokenization_spaces=False,
|
clean_up_tokenization_spaces=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
return output_text[0].strip()
|
caption = output_text[0]
|
||||||
|
# thinking models (e.g. Qwen3.6) may still emit reasoning before the
|
||||||
|
# answer; keep only what follows the think block
|
||||||
|
if "</think>" in caption:
|
||||||
|
caption = caption.split("</think>")[-1]
|
||||||
|
return caption.strip()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"Error processing {file_path}: {e}")
|
print(f"Error processing {file_path}: {e}")
|
||||||
traceback.print_exc()
|
traceback.print_exc()
|
||||||
|
|||||||
@@ -183,6 +183,13 @@ const CaptionSimpleJob: React.FC<Props> = ({ jobConfig, setJobConfig, gpuIDs, se
|
|||||||
checked={jobConfig.config.process[0].caption.compile || false}
|
checked={jobConfig.config.process[0].caption.compile || false}
|
||||||
onChange={value => setJobConfig(value, 'config.process[0].caption.compile')}
|
onChange={value => setJobConfig(value, 'config.process[0].caption.compile')}
|
||||||
/>
|
/>
|
||||||
|
{additionalSections.includes('caption.thinking') && (
|
||||||
|
<Checkbox
|
||||||
|
label="Thinking"
|
||||||
|
checked={jobConfig.config.process[0].caption.thinking || false}
|
||||||
|
onChange={value => setJobConfig(value, 'config.process[0].caption.thinking')}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
</FormGroup>
|
</FormGroup>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import { GroupedSelectOption, SelectOption } from "@/types";
|
import { GroupedSelectOption, SelectOption } from "@/types";
|
||||||
|
|
||||||
type CaptionGroup = 'image' | 'music';
|
type CaptionGroup = 'image' | 'music';
|
||||||
type AdditionalSections = 'caption.model_name_or_path2' | 'caption.caption_prompt' | 'caption.max_res' | 'caption.max_new_tokens' | 'caption.fixed_caption';
|
type AdditionalSections = 'caption.model_name_or_path2' | 'caption.caption_prompt' | 'caption.max_res' | 'caption.max_new_tokens' | 'caption.fixed_caption' | 'caption.thinking';
|
||||||
|
|
||||||
export interface CaptionOption {
|
export interface CaptionOption {
|
||||||
name: string;
|
name: string;
|
||||||
@@ -66,12 +66,16 @@ export const captionerTypes: CaptionOption[] = [
|
|||||||
{ value: 'Qwen/Qwen3-VL-2B-Instruct', label: 'Qwen/Qwen3-VL-2B-Instruct' },
|
{ value: 'Qwen/Qwen3-VL-2B-Instruct', label: 'Qwen/Qwen3-VL-2B-Instruct' },
|
||||||
{ value: 'Qwen/Qwen3-VL-4B-Instruct', label: 'Qwen/Qwen3-VL-4B-Instruct' },
|
{ value: 'Qwen/Qwen3-VL-4B-Instruct', label: 'Qwen/Qwen3-VL-4B-Instruct' },
|
||||||
{ value: 'Qwen/Qwen3-VL-8B-Instruct', label: 'Qwen/Qwen3-VL-8B-Instruct' },
|
{ value: 'Qwen/Qwen3-VL-8B-Instruct', label: 'Qwen/Qwen3-VL-8B-Instruct' },
|
||||||
|
{ value: 'huihui-ai/Huihui-Qwen3-VL-8B-Instruct-abliterated', label: 'huihui-ai/Huihui-Qwen3-VL-8B-Instruct-abliterated' },
|
||||||
{ value: 'Qwen/Qwen3-VL-30B-A3B-Instruct', label: 'Qwen/Qwen3-VL-30B-A3B-Instruct' },
|
{ value: 'Qwen/Qwen3-VL-30B-A3B-Instruct', label: 'Qwen/Qwen3-VL-30B-A3B-Instruct' },
|
||||||
|
{ value: 'Qwen/Qwen3.6-27B', label: 'Qwen/Qwen3.6-27B' },
|
||||||
|
{ value: 'huihui-ai/Huihui-Qwen3.6-27B-abliterated', label: 'huihui-ai/Huihui-Qwen3.6-27B-abliterated' },
|
||||||
],
|
],
|
||||||
additionalSections: [
|
additionalSections: [
|
||||||
'caption.caption_prompt',
|
'caption.caption_prompt',
|
||||||
'caption.max_res',
|
'caption.max_res',
|
||||||
'caption.max_new_tokens',
|
'caption.max_new_tokens',
|
||||||
|
'caption.thinking',
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -281,6 +281,7 @@ export interface CaptionProcessConfig {
|
|||||||
max_new_tokens?: number;
|
max_new_tokens?: number;
|
||||||
fixed_caption?: string;
|
fixed_caption?: string;
|
||||||
caption_extension?: string;
|
caption_extension?: string;
|
||||||
|
thinking?: boolean;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user