Add a projection layer on vision direct when doing image embeds
This commit is contained in:
@@ -754,6 +754,9 @@ class VisionDirectAdapter(torch.nn.Module):
|
|||||||
output_dim=self.config.sparse_autoencoder_dim
|
output_dim=self.config.sparse_autoencoder_dim
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if self.config.clip_layer == "image_embeds":
|
||||||
|
self.proj = nn.Linear(self.token_size, self.token_size)
|
||||||
|
|
||||||
def state_dict(self, destination=None, prefix='', keep_vars=False):
|
def state_dict(self, destination=None, prefix='', keep_vars=False):
|
||||||
if self.config.train_scaler:
|
if self.config.train_scaler:
|
||||||
# only return the block scaler
|
# only return the block scaler
|
||||||
@@ -777,6 +780,7 @@ class VisionDirectAdapter(torch.nn.Module):
|
|||||||
# if doing image_embeds, normalize here
|
# if doing image_embeds, normalize here
|
||||||
if self.config.clip_layer == "image_embeds":
|
if self.config.clip_layer == "image_embeds":
|
||||||
input = norm_layer(input)
|
input = norm_layer(input)
|
||||||
|
input = self.proj(input)
|
||||||
if self.resampler is not None:
|
if self.resampler is not None:
|
||||||
input = self.resampler(input)
|
input = self.resampler(input)
|
||||||
if self.pool is not None:
|
if self.pool is not None:
|
||||||
|
|||||||
Reference in New Issue
Block a user