Phase 1
This commit is contained in:
Jaret Burkett
10
toolkit/models/v2/vae/autoencoder_kl.py
Normal file
10
toolkit/models/v2/vae/autoencoder_kl.py
Normal file
@@ -0,0 +1,10 @@
|
||||
from diffusers import AutoencoderKL
|
||||
|
||||
from .._mixin import OstrisModelMixin
|
||||
|
||||
|
||||
class KLVAE(AutoencoderKL, OstrisModelMixin):
|
||||
"""The diffusers AutoencoderKL (SD/SDXL/Flux1/Z-Image image VAEs), loaded
|
||||
from a checkpoint's vae/ subfolder through the universal loader."""
|
||||
|
||||
aitk_subfolder = "vae"
|
||||
539
toolkit/models/v2/vae/flux2_kl.py
Normal file
539
toolkit/models/v2/vae/flux2_kl.py
Normal file
@@ -0,0 +1,539 @@
|
||||
"""The BFL-style Flux2 KL autoencoder (32ch latents, 2x2 pixel-shuffle
|
||||
packing to 128ch), shared by the flux2 family and ideogram4.
|
||||
|
||||
flux2 loads the raw BFL ae.safetensors layout and uses encode/decode (with the
|
||||
BatchNorm running-stats latent normalization); ideogram4 loads diffusers-format
|
||||
checkpoints via convert_diffusers_state_dict and drives encoder/decoder
|
||||
directly with its own patchify + latent-norm tables. The two models pack the
|
||||
128 latent channels in different orders — the packing/normalization stays
|
||||
per-model.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import torch
|
||||
import torch.utils.checkpoint as ckpt
|
||||
from einops import rearrange
|
||||
from torch import Tensor, nn
|
||||
|
||||
|
||||
@dataclass
|
||||
class AutoEncoderParams:
|
||||
resolution: int = 256
|
||||
in_channels: int = 3
|
||||
ch: int = 128
|
||||
out_ch: int = 3
|
||||
ch_mult: list[int] = field(default_factory=lambda: [1, 2, 4, 4])
|
||||
num_res_blocks: int = 2
|
||||
z_channels: int = 32
|
||||
|
||||
@dataclass
|
||||
class AutoEncoderSmallDecoderParams:
|
||||
resolution: int = 256
|
||||
in_channels: int = 3
|
||||
ch: int = 128
|
||||
ch_encoder: int = 96
|
||||
out_ch: int = 3
|
||||
ch_mult: list[int] = field(default_factory=lambda: [1, 2, 4, 4])
|
||||
num_res_blocks: int = 2
|
||||
z_channels: int = 32
|
||||
|
||||
|
||||
def swish(x: Tensor) -> Tensor:
|
||||
return x * torch.sigmoid(x)
|
||||
|
||||
|
||||
class AttnBlock(nn.Module):
|
||||
def __init__(self, in_channels: int):
|
||||
super().__init__()
|
||||
self.in_channels = in_channels
|
||||
|
||||
self.norm = nn.GroupNorm(
|
||||
num_groups=32, num_channels=in_channels, eps=1e-6, affine=True
|
||||
)
|
||||
|
||||
self.q = nn.Conv2d(in_channels, in_channels, kernel_size=1)
|
||||
self.k = nn.Conv2d(in_channels, in_channels, kernel_size=1)
|
||||
self.v = nn.Conv2d(in_channels, in_channels, kernel_size=1)
|
||||
self.proj_out = nn.Conv2d(in_channels, in_channels, kernel_size=1)
|
||||
|
||||
def attention(self, h_: Tensor) -> Tensor:
|
||||
h_ = self.norm(h_)
|
||||
q = self.q(h_)
|
||||
k = self.k(h_)
|
||||
v = self.v(h_)
|
||||
|
||||
b, c, h, w = q.shape
|
||||
q = rearrange(q, "b c h w -> b 1 (h w) c").contiguous()
|
||||
k = rearrange(k, "b c h w -> b 1 (h w) c").contiguous()
|
||||
v = rearrange(v, "b c h w -> b 1 (h w) c").contiguous()
|
||||
h_ = nn.functional.scaled_dot_product_attention(q, k, v)
|
||||
|
||||
return rearrange(h_, "b 1 (h w) c -> b c h w", h=h, w=w, c=c, b=b)
|
||||
|
||||
def forward(self, x: Tensor) -> Tensor:
|
||||
return x + self.proj_out(self.attention(x))
|
||||
|
||||
|
||||
class ResnetBlock(nn.Module):
|
||||
def __init__(self, in_channels: int, out_channels: int):
|
||||
super().__init__()
|
||||
self.in_channels = in_channels
|
||||
out_channels = in_channels if out_channels is None else out_channels
|
||||
self.out_channels = out_channels
|
||||
|
||||
self.norm1 = nn.GroupNorm(
|
||||
num_groups=32, num_channels=in_channels, eps=1e-6, affine=True
|
||||
)
|
||||
self.conv1 = nn.Conv2d(
|
||||
in_channels, out_channels, kernel_size=3, stride=1, padding=1
|
||||
)
|
||||
self.norm2 = nn.GroupNorm(
|
||||
num_groups=32, num_channels=out_channels, eps=1e-6, affine=True
|
||||
)
|
||||
self.conv2 = nn.Conv2d(
|
||||
out_channels, out_channels, kernel_size=3, stride=1, padding=1
|
||||
)
|
||||
if self.in_channels != self.out_channels:
|
||||
self.nin_shortcut = nn.Conv2d(
|
||||
in_channels, out_channels, kernel_size=1, stride=1, padding=0
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
h = x
|
||||
h = self.norm1(h)
|
||||
h = swish(h)
|
||||
h = self.conv1(h)
|
||||
|
||||
h = self.norm2(h)
|
||||
h = swish(h)
|
||||
h = self.conv2(h)
|
||||
|
||||
if self.in_channels != self.out_channels:
|
||||
x = self.nin_shortcut(x)
|
||||
|
||||
return x + h
|
||||
|
||||
|
||||
class Downsample(nn.Module):
|
||||
def __init__(self, in_channels: int):
|
||||
super().__init__()
|
||||
# no asymmetric padding in torch conv, must do it ourselves
|
||||
self.conv = nn.Conv2d(
|
||||
in_channels, in_channels, kernel_size=3, stride=2, padding=0
|
||||
)
|
||||
|
||||
def forward(self, x: Tensor):
|
||||
pad = (0, 1, 0, 1)
|
||||
x = nn.functional.pad(x, pad, mode="constant", value=0)
|
||||
x = self.conv(x)
|
||||
return x
|
||||
|
||||
|
||||
class Upsample(nn.Module):
|
||||
def __init__(self, in_channels: int):
|
||||
super().__init__()
|
||||
self.conv = nn.Conv2d(
|
||||
in_channels, in_channels, kernel_size=3, stride=1, padding=1
|
||||
)
|
||||
|
||||
def forward(self, x: Tensor):
|
||||
x = nn.functional.interpolate(x, scale_factor=2.0, mode="nearest")
|
||||
x = self.conv(x)
|
||||
return x
|
||||
|
||||
|
||||
class Encoder(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
resolution: int,
|
||||
in_channels: int,
|
||||
ch: int,
|
||||
ch_mult: list[int],
|
||||
num_res_blocks: int,
|
||||
z_channels: int,
|
||||
):
|
||||
super().__init__()
|
||||
self.quant_conv = torch.nn.Conv2d(2 * z_channels, 2 * z_channels, 1)
|
||||
self.ch = ch
|
||||
self.num_resolutions = len(ch_mult)
|
||||
self.num_res_blocks = num_res_blocks
|
||||
self.resolution = resolution
|
||||
self.in_channels = in_channels
|
||||
# downsampling
|
||||
self.conv_in = nn.Conv2d(
|
||||
in_channels, self.ch, kernel_size=3, stride=1, padding=1
|
||||
)
|
||||
|
||||
curr_res = resolution
|
||||
in_ch_mult = (1,) + tuple(ch_mult)
|
||||
self.in_ch_mult = in_ch_mult
|
||||
self.down = nn.ModuleList()
|
||||
block_in = self.ch
|
||||
for i_level in range(self.num_resolutions):
|
||||
block = nn.ModuleList()
|
||||
attn = nn.ModuleList()
|
||||
block_in = ch * in_ch_mult[i_level]
|
||||
block_out = ch * ch_mult[i_level]
|
||||
for _ in range(self.num_res_blocks):
|
||||
block.append(ResnetBlock(in_channels=block_in, out_channels=block_out))
|
||||
block_in = block_out
|
||||
down = nn.Module()
|
||||
down.block = block
|
||||
down.attn = attn
|
||||
if i_level != self.num_resolutions - 1:
|
||||
down.downsample = Downsample(block_in)
|
||||
curr_res = curr_res // 2
|
||||
self.down.append(down)
|
||||
|
||||
# middle
|
||||
self.mid = nn.Module()
|
||||
self.mid.block_1 = ResnetBlock(in_channels=block_in, out_channels=block_in)
|
||||
self.mid.attn_1 = AttnBlock(block_in)
|
||||
self.mid.block_2 = ResnetBlock(in_channels=block_in, out_channels=block_in)
|
||||
|
||||
# end
|
||||
self.norm_out = nn.GroupNorm(
|
||||
num_groups=32, num_channels=block_in, eps=1e-6, affine=True
|
||||
)
|
||||
self.conv_out = nn.Conv2d(
|
||||
block_in, 2 * z_channels, kernel_size=3, stride=1, padding=1
|
||||
)
|
||||
self.gradient_checkpointing = False
|
||||
|
||||
def enable_gradient_checkpointing(self):
|
||||
self.gradient_checkpointing = True
|
||||
|
||||
def forward(self, x: Tensor) -> Tensor:
|
||||
# downsampling
|
||||
hs = [self.conv_in(x)]
|
||||
for i_level in range(self.num_resolutions):
|
||||
for i_block in range(self.num_res_blocks):
|
||||
if torch.is_grad_enabled() and self.gradient_checkpointing:
|
||||
h = ckpt.checkpoint(self.down[i_level].block[i_block], hs[-1])
|
||||
if len(self.down[i_level].attn) > 0:
|
||||
h = ckpt.checkpoint(self.down[i_level].attn[i_block], h)
|
||||
else:
|
||||
h = self.down[i_level].block[i_block](hs[-1])
|
||||
if len(self.down[i_level].attn) > 0:
|
||||
h = self.down[i_level].attn[i_block](h)
|
||||
hs.append(h)
|
||||
if i_level != self.num_resolutions - 1:
|
||||
if torch.is_grad_enabled() and self.gradient_checkpointing:
|
||||
hs.append(ckpt.checkpoint(self.down[i_level].downsample, hs[-1]))
|
||||
else:
|
||||
hs.append(self.down[i_level].downsample(hs[-1]))
|
||||
|
||||
# middle
|
||||
h = hs[-1]
|
||||
if torch.is_grad_enabled() and self.gradient_checkpointing:
|
||||
h = ckpt.checkpoint(self.mid.block_1, h)
|
||||
h = ckpt.checkpoint(self.mid.attn_1, h)
|
||||
h = ckpt.checkpoint(self.mid.block_2, h)
|
||||
else:
|
||||
h = self.mid.block_1(h)
|
||||
h = self.mid.attn_1(h)
|
||||
h = self.mid.block_2(h)
|
||||
# end
|
||||
h = self.norm_out(h)
|
||||
h = swish(h)
|
||||
h = self.conv_out(h)
|
||||
h = self.quant_conv(h)
|
||||
return h
|
||||
|
||||
|
||||
class Decoder(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
ch: int,
|
||||
out_ch: int,
|
||||
ch_mult: list[int],
|
||||
num_res_blocks: int,
|
||||
in_channels: int,
|
||||
resolution: int,
|
||||
z_channels: int,
|
||||
):
|
||||
super().__init__()
|
||||
self.post_quant_conv = torch.nn.Conv2d(z_channels, z_channels, 1)
|
||||
self.ch = ch
|
||||
self.num_resolutions = len(ch_mult)
|
||||
self.num_res_blocks = num_res_blocks
|
||||
self.resolution = resolution
|
||||
self.in_channels = in_channels
|
||||
self.ffactor = 2 ** (self.num_resolutions - 1)
|
||||
|
||||
# compute in_ch_mult, block_in and curr_res at lowest res
|
||||
block_in = ch * ch_mult[self.num_resolutions - 1]
|
||||
curr_res = resolution // 2 ** (self.num_resolutions - 1)
|
||||
self.z_shape = (1, z_channels, curr_res, curr_res)
|
||||
|
||||
# z to block_in
|
||||
self.conv_in = nn.Conv2d(
|
||||
z_channels, block_in, kernel_size=3, stride=1, padding=1
|
||||
)
|
||||
|
||||
# middle
|
||||
self.mid = nn.Module()
|
||||
self.mid.block_1 = ResnetBlock(in_channels=block_in, out_channels=block_in)
|
||||
self.mid.attn_1 = AttnBlock(block_in)
|
||||
self.mid.block_2 = ResnetBlock(in_channels=block_in, out_channels=block_in)
|
||||
|
||||
# upsampling
|
||||
self.up = nn.ModuleList()
|
||||
for i_level in reversed(range(self.num_resolutions)):
|
||||
block = nn.ModuleList()
|
||||
attn = nn.ModuleList()
|
||||
block_out = ch * ch_mult[i_level]
|
||||
for _ in range(self.num_res_blocks + 1):
|
||||
block.append(ResnetBlock(in_channels=block_in, out_channels=block_out))
|
||||
block_in = block_out
|
||||
up = nn.Module()
|
||||
up.block = block
|
||||
up.attn = attn
|
||||
if i_level != 0:
|
||||
up.upsample = Upsample(block_in)
|
||||
curr_res = curr_res * 2
|
||||
self.up.insert(0, up) # prepend to get consistent order
|
||||
|
||||
# end
|
||||
self.norm_out = nn.GroupNorm(
|
||||
num_groups=32, num_channels=block_in, eps=1e-6, affine=True
|
||||
)
|
||||
self.conv_out = nn.Conv2d(block_in, out_ch, kernel_size=3, stride=1, padding=1)
|
||||
self.gradient_checkpointing = False
|
||||
|
||||
def enable_gradient_checkpointing(self):
|
||||
self.gradient_checkpointing = True
|
||||
|
||||
def forward(self, z: Tensor) -> Tensor:
|
||||
z = self.post_quant_conv(z)
|
||||
|
||||
# get dtype for proper tracing
|
||||
upscale_dtype = next(self.up.parameters()).dtype
|
||||
|
||||
# z to block_in
|
||||
h = self.conv_in(z)
|
||||
|
||||
# middle
|
||||
if torch.is_grad_enabled() and self.gradient_checkpointing:
|
||||
h = ckpt.checkpoint(self.mid.block_1, h)
|
||||
h = ckpt.checkpoint(self.mid.attn_1, h)
|
||||
h = ckpt.checkpoint(self.mid.block_2, h)
|
||||
else:
|
||||
h = self.mid.block_1(h)
|
||||
h = self.mid.attn_1(h)
|
||||
h = self.mid.block_2(h)
|
||||
|
||||
# cast to proper dtype
|
||||
h = h.to(upscale_dtype)
|
||||
# upsampling
|
||||
for i_level in reversed(range(self.num_resolutions)):
|
||||
for i_block in range(self.num_res_blocks + 1):
|
||||
if torch.is_grad_enabled() and self.gradient_checkpointing:
|
||||
h = ckpt.checkpoint(self.up[i_level].block[i_block], h)
|
||||
if len(self.up[i_level].attn) > 0:
|
||||
h = ckpt.checkpoint(self.up[i_level].attn[i_block], h)
|
||||
else:
|
||||
h = self.up[i_level].block[i_block](h)
|
||||
if len(self.up[i_level].attn) > 0:
|
||||
h = self.up[i_level].attn[i_block](h)
|
||||
if i_level != 0:
|
||||
if torch.is_grad_enabled() and self.gradient_checkpointing:
|
||||
h = ckpt.checkpoint(self.up[i_level].upsample, h)
|
||||
else:
|
||||
h = self.up[i_level].upsample(h)
|
||||
|
||||
# end
|
||||
h = self.norm_out(h)
|
||||
h = swish(h)
|
||||
h = self.conv_out(h)
|
||||
return h
|
||||
|
||||
|
||||
class AutoEncoder(nn.Module):
|
||||
def __init__(self, params: AutoEncoderParams):
|
||||
super().__init__()
|
||||
self.params = params
|
||||
self.encoder = Encoder(
|
||||
resolution=params.resolution,
|
||||
in_channels=params.in_channels,
|
||||
ch=params.ch,
|
||||
ch_mult=params.ch_mult,
|
||||
num_res_blocks=params.num_res_blocks,
|
||||
z_channels=params.z_channels,
|
||||
)
|
||||
decoder_ch = params.ch
|
||||
if hasattr(params, "ch_encoder"):
|
||||
decoder_ch = params.ch_encoder
|
||||
self.decoder = Decoder(
|
||||
resolution=params.resolution,
|
||||
in_channels=params.in_channels,
|
||||
ch=decoder_ch,
|
||||
out_ch=params.out_ch,
|
||||
ch_mult=params.ch_mult,
|
||||
num_res_blocks=params.num_res_blocks,
|
||||
z_channels=params.z_channels,
|
||||
)
|
||||
|
||||
self.bn_eps = 1e-4
|
||||
self.bn_momentum = 0.1
|
||||
self.ps = [2, 2]
|
||||
self.bn = torch.nn.BatchNorm2d(
|
||||
math.prod(self.ps) * params.z_channels,
|
||||
eps=self.bn_eps,
|
||||
momentum=self.bn_momentum,
|
||||
affine=False,
|
||||
track_running_stats=True,
|
||||
)
|
||||
self._gradient_checkpointing = False
|
||||
|
||||
@property
|
||||
def gradient_checkpointing(self):
|
||||
return self._gradient_checkpointing
|
||||
|
||||
@gradient_checkpointing.setter
|
||||
def gradient_checkpointing(self, value: bool):
|
||||
self._gradient_checkpointing = value
|
||||
self.encoder.gradient_checkpointing = value
|
||||
self.decoder.gradient_checkpointing = value
|
||||
|
||||
@property
|
||||
def device(self):
|
||||
return next(self.parameters()).device
|
||||
|
||||
@property
|
||||
def dtype(self):
|
||||
return next(self.parameters()).dtype
|
||||
|
||||
def enable_gradient_checkpointing(self):
|
||||
self.gradient_checkpointing = True
|
||||
self.encoder.enable_gradient_checkpointing()
|
||||
self.decoder.enable_gradient_checkpointing()
|
||||
|
||||
def normalize(self, z):
|
||||
self.bn.eval()
|
||||
return self.bn(z)
|
||||
|
||||
def inv_normalize(self, z):
|
||||
self.bn.eval()
|
||||
s = torch.sqrt(self.bn.running_var.view(1, -1, 1, 1) + self.bn_eps)
|
||||
m = self.bn.running_mean.view(1, -1, 1, 1)
|
||||
return z * s + m
|
||||
|
||||
def encode(self, x: Tensor) -> Tensor:
|
||||
moments = self.encoder(x)
|
||||
mean = torch.chunk(moments, 2, dim=1)[0]
|
||||
|
||||
z = rearrange(
|
||||
mean,
|
||||
"... c (i pi) (j pj) -> ... (c pi pj) i j",
|
||||
pi=self.ps[0],
|
||||
pj=self.ps[1],
|
||||
)
|
||||
z = self.normalize(z)
|
||||
return z
|
||||
|
||||
def decode(self, z: Tensor) -> Tensor:
|
||||
z = self.inv_normalize(z)
|
||||
z = rearrange(
|
||||
z,
|
||||
"... (c pi pj) i j -> ... c (i pi) (j pj)",
|
||||
pi=self.ps[0],
|
||||
pj=self.ps[1],
|
||||
)
|
||||
dec = self.decoder(z)
|
||||
return dec
|
||||
|
||||
|
||||
_NUM_RESOLUTIONS = 4
|
||||
|
||||
|
||||
def convert_diffusers_state_dict(src: dict[str, Tensor]) -> dict[str, Tensor]:
|
||||
out: dict[str, Tensor] = {}
|
||||
attn_substrings = (".mid.attn_1.",)
|
||||
for src_key, tensor in src.items():
|
||||
dst_key = _rewrite_diffusers_key(src_key)
|
||||
if dst_key is None:
|
||||
raise KeyError(f"Unrecognized diffusers VAE state-dict key: {src_key}")
|
||||
if (
|
||||
any(s in dst_key for s in attn_substrings)
|
||||
and dst_key.endswith(".weight")
|
||||
and tensor.ndim == 2
|
||||
):
|
||||
tensor = tensor.unsqueeze(-1).unsqueeze(-1)
|
||||
out[dst_key] = tensor
|
||||
return out
|
||||
|
||||
|
||||
def _rewrite_diffusers_key(key: str) -> str | None:
|
||||
if key.startswith("bn."):
|
||||
return key
|
||||
|
||||
if key.startswith("quant_conv."):
|
||||
return key.replace("quant_conv.", "encoder.quant_conv.", 1)
|
||||
if key.startswith("post_quant_conv."):
|
||||
return key.replace("post_quant_conv.", "decoder.post_quant_conv.", 1)
|
||||
|
||||
if key == "encoder.conv_norm_out.weight":
|
||||
return "encoder.norm_out.weight"
|
||||
if key == "encoder.conv_norm_out.bias":
|
||||
return "encoder.norm_out.bias"
|
||||
if key == "decoder.conv_norm_out.weight":
|
||||
return "decoder.norm_out.weight"
|
||||
if key == "decoder.conv_norm_out.bias":
|
||||
return "decoder.norm_out.bias"
|
||||
|
||||
m = re.match(r"^(encoder|decoder)\.mid_block\.resnets\.(\d+)\.(.+)$", key)
|
||||
if m:
|
||||
side, idx, rest = m.group(1), int(m.group(2)), m.group(3)
|
||||
rest = rest.replace("conv_shortcut", "nin_shortcut")
|
||||
return f"{side}.mid.block_{idx + 1}.{rest}"
|
||||
m = re.match(r"^(encoder|decoder)\.mid_block\.attentions\.0\.(.+)$", key)
|
||||
if m:
|
||||
side, rest = m.group(1), m.group(2)
|
||||
rest = (
|
||||
rest.replace("group_norm.", "norm.")
|
||||
.replace("to_q.", "q.")
|
||||
.replace("to_k.", "k.")
|
||||
.replace("to_v.", "v.")
|
||||
.replace("to_out.0.", "proj_out.")
|
||||
)
|
||||
return f"{side}.mid.attn_1.{rest}"
|
||||
|
||||
m = re.match(r"^encoder\.down_blocks\.(\d+)\.resnets\.(\d+)\.(.+)$", key)
|
||||
if m:
|
||||
level, res_idx, rest = m.group(1), m.group(2), m.group(3)
|
||||
rest = rest.replace("conv_shortcut", "nin_shortcut")
|
||||
return f"encoder.down.{level}.block.{res_idx}.{rest}"
|
||||
m = re.match(r"^encoder\.down_blocks\.(\d+)\.downsamplers\.0\.conv\.(.+)$", key)
|
||||
if m:
|
||||
return f"encoder.down.{m.group(1)}.downsample.conv.{m.group(2)}"
|
||||
|
||||
m = re.match(r"^decoder\.up_blocks\.(\d+)\.resnets\.(\d+)\.(.+)$", key)
|
||||
if m:
|
||||
diffusers_idx = int(m.group(1))
|
||||
res_idx = m.group(2)
|
||||
rest = m.group(3).replace("conv_shortcut", "nin_shortcut")
|
||||
return (
|
||||
f"decoder.up.{_NUM_RESOLUTIONS - 1 - diffusers_idx}.block.{res_idx}.{rest}"
|
||||
)
|
||||
m = re.match(r"^decoder\.up_blocks\.(\d+)\.upsamplers\.0\.conv\.(.+)$", key)
|
||||
if m:
|
||||
diffusers_idx = int(m.group(1))
|
||||
return f"decoder.up.{_NUM_RESOLUTIONS - 1 - diffusers_idx}.upsample.conv.{m.group(2)}"
|
||||
|
||||
if key.startswith(
|
||||
(
|
||||
"encoder.conv_in.",
|
||||
"encoder.conv_out.",
|
||||
"decoder.conv_in.",
|
||||
"decoder.conv_out.",
|
||||
)
|
||||
):
|
||||
return key
|
||||
|
||||
return None
|
||||
86
toolkit/models/v2/vae/qwen_image.py
Normal file
86
toolkit/models/v2/vae/qwen_image.py
Normal file
@@ -0,0 +1,86 @@
|
||||
import torch
|
||||
from diffusers import AutoencoderKLQwenImage
|
||||
|
||||
from .._mixin import OstrisModelMixin
|
||||
|
||||
|
||||
class QwenImageVAE(AutoencoderKLQwenImage, OstrisModelMixin):
|
||||
"""The Qwen-Image (wan-style video) VAE, shared by the qwen_image family,
|
||||
nucleus_image and krea2."""
|
||||
|
||||
aitk_subfolder = "vae"
|
||||
|
||||
|
||||
class QwenImageVAEHolderMixin:
|
||||
"""BaseModel-side encode_images/decode_latents for models whose self.vae
|
||||
is the Qwen-Image VAE: it is a video VAE, so images ride in a single-frame
|
||||
dim and latents are normalized with the config's latents_mean/std."""
|
||||
|
||||
# tile the decode when low_vram (decode only; encode stays untiled)
|
||||
vae_decode_tiled_on_low_vram = False
|
||||
|
||||
def encode_images(self, image_list, device=None, dtype=None):
|
||||
if device is None:
|
||||
device = self.vae_device_torch
|
||||
if dtype is None:
|
||||
dtype = self.vae_torch_dtype
|
||||
|
||||
if self.vae.device == torch.device("cpu"):
|
||||
self.vae.to(device)
|
||||
self.vae.eval()
|
||||
self.vae.requires_grad_(False)
|
||||
|
||||
image_list = [image.to(device, dtype=dtype) for image in image_list]
|
||||
images = torch.stack(image_list).to(device, dtype=dtype)
|
||||
images = images.unsqueeze(2) # add the frame dim
|
||||
latents = self.vae.encode(images).latent_dist.sample()
|
||||
|
||||
latents_mean = (
|
||||
torch.tensor(self.vae.config.latents_mean)
|
||||
.view(1, self.vae.config.z_dim, 1, 1, 1)
|
||||
.to(latents.device, latents.dtype)
|
||||
)
|
||||
latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(
|
||||
1, self.vae.config.z_dim, 1, 1, 1
|
||||
).to(latents.device, latents.dtype)
|
||||
|
||||
latents = (latents - latents_mean) * latents_std
|
||||
latents = latents.squeeze(2) # drop the frame dim
|
||||
return latents.to(device, dtype=dtype)
|
||||
|
||||
def decode_latents(self, latents: torch.Tensor, device=None, dtype=None):
|
||||
if device is None:
|
||||
device = self.vae_device_torch
|
||||
if dtype is None:
|
||||
dtype = self.vae_torch_dtype
|
||||
|
||||
if self.vae.device == torch.device("cpu"):
|
||||
self.vae.to(device)
|
||||
|
||||
latents = latents.to(device, dtype=dtype)
|
||||
latents = latents.unsqueeze(2) # add the frame dim
|
||||
|
||||
latents_mean = (
|
||||
torch.tensor(self.vae.config.latents_mean)
|
||||
.view(1, self.vae.config.z_dim, 1, 1, 1)
|
||||
.to(latents.device, latents.dtype)
|
||||
)
|
||||
latents_std = (
|
||||
torch.tensor(self.vae.config.latents_std)
|
||||
.view(1, self.vae.config.z_dim, 1, 1, 1)
|
||||
.to(latents.device, latents.dtype)
|
||||
)
|
||||
latents = latents * latents_std + latents_mean
|
||||
|
||||
# full-resolution decode spikes VRAM; models opt in to tiling it
|
||||
tiled = self.vae_decode_tiled_on_low_vram and self.model_config.low_vram
|
||||
if tiled:
|
||||
self.vae.enable_tiling()
|
||||
try:
|
||||
images = self.vae.decode(latents).sample
|
||||
finally:
|
||||
if tiled:
|
||||
self.vae.disable_tiling()
|
||||
|
||||
images = images.squeeze(2) # drop the frame dim
|
||||
return images.to(device, dtype=dtype)
|
||||
Reference in New Issue
Block a user