Added some more useful error handeling and logging
This commit is contained in:
1 parent
7c21eac1b3
commit
6c8b5ab606
2 files changed
+29
-2
No files matched your search
@@ -7,6 +7,7 @@ import shutil
|
|||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
import traceback
|
||||||
from typing import Union, List, Optional
|
from typing import Union, List, Optional
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
@@ -2008,7 +2009,17 @@ class BaseSDTrainProcess(BaseTrainProcess):
|
|||||||
# flush()
|
# flush()
|
||||||
### HOOK ###
|
### HOOK ###
|
||||||
with self.accelerator.accumulate(self.modules_being_trained):
|
with self.accelerator.accumulate(self.modules_being_trained):
|
||||||
|
try:
|
||||||
loss_dict = self.hook_train_loop(batch_list)
|
loss_dict = self.hook_train_loop(batch_list)
|
||||||
|
except Exception as e:
|
||||||
|
traceback.print_exc()
|
||||||
|
#print batch info
|
||||||
|
print("Batch Items:")
|
||||||
|
for batch in batch_list:
|
||||||
|
for item in batch.file_items:
|
||||||
|
print(f" - {item.path}")
|
||||||
|
raise e
|
||||||
|
|
||||||
self.timer.stop('train_loop')
|
self.timer.stop('train_loop')
|
||||||
if not did_first_flush:
|
if not did_first_flush:
|
||||||
flush()
|
flush()
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ from diffusers import Transformer2DModel, FluxTransformer2DModel
|
|||||||
from transformers import T5EncoderModel, CLIPTextModel, CLIPTokenizer, T5Tokenizer, CLIPVisionModelWithProjection
|
from transformers import T5EncoderModel, CLIPTextModel, CLIPTokenizer, T5Tokenizer, CLIPVisionModelWithProjection
|
||||||
from toolkit.models.pixtral_vision import PixtralVisionEncoder, PixtralVisionImagePreprocessor, VisionLanguageAdapter
|
from toolkit.models.pixtral_vision import PixtralVisionEncoder, PixtralVisionImagePreprocessor, VisionLanguageAdapter
|
||||||
from transformers import SiglipImageProcessor, SiglipVisionModel
|
from transformers import SiglipImageProcessor, SiglipVisionModel
|
||||||
|
import traceback
|
||||||
from toolkit.config_modules import AdapterConfig
|
from toolkit.config_modules import AdapterConfig
|
||||||
from toolkit.paths import REPOS_ROOT
|
from toolkit.paths import REPOS_ROOT
|
||||||
sys.path.append(REPOS_ROOT)
|
sys.path.append(REPOS_ROOT)
|
||||||
@@ -291,9 +291,10 @@ class VisionDirectAdapterAttnProcessor(nn.Module):
|
|||||||
|
|
||||||
# only use one TE or the other. If our adapter is active only use ours
|
# only use one TE or the other. If our adapter is active only use ours
|
||||||
if self.is_active and self.conditional_embeds is not None:
|
if self.is_active and self.conditional_embeds is not None:
|
||||||
|
try:
|
||||||
|
|
||||||
adapter_hidden_states = self.conditional_embeds
|
adapter_hidden_states = self.conditional_embeds
|
||||||
if adapter_hidden_states.shape[0] < batch_size:
|
if adapter_hidden_states.shape[0] == batch_size // 2:
|
||||||
adapter_hidden_states = torch.cat([
|
adapter_hidden_states = torch.cat([
|
||||||
self.unconditional_embeds,
|
self.unconditional_embeds,
|
||||||
adapter_hidden_states
|
adapter_hidden_states
|
||||||
@@ -321,6 +322,21 @@ class VisionDirectAdapterAttnProcessor(nn.Module):
|
|||||||
vd_hidden_states = vd_hidden_states.to(query.dtype)
|
vd_hidden_states = vd_hidden_states.to(query.dtype)
|
||||||
|
|
||||||
hidden_states = hidden_states + self.scale * vd_hidden_states
|
hidden_states = hidden_states + self.scale * vd_hidden_states
|
||||||
|
except Exception as e:
|
||||||
|
print("Error in VisionDirectAdapterAttnProcessor")
|
||||||
|
# print shapes of all tensors
|
||||||
|
print(f"hidden_states: {hidden_states.shape}")
|
||||||
|
print(f"adapter_hidden_states: {adapter_hidden_states.shape}")
|
||||||
|
print(f"vd_key: {vd_key.shape}")
|
||||||
|
print(f"vd_value: {vd_value.shape}")
|
||||||
|
print(f"vd_hidden_states: {vd_hidden_states.shape}")
|
||||||
|
print(f"query: {query.shape}")
|
||||||
|
print(f"key: {key.shape}")
|
||||||
|
print(f"value: {value.shape}")
|
||||||
|
print(f"inner_dim: {inner_dim}")
|
||||||
|
print(f"head_dim: {head_dim}")
|
||||||
|
print(f"batch_size: {batch_size}")
|
||||||
|
traceback.print_exc()
|
||||||
|
|
||||||
|
|
||||||
# linear proj
|
# linear proj
|
||||||
|
|||||||
Reference in new issue
Block a user