Stability improvements to model offloading. Added D-OPSD bleed loss as well.

This commit is contained in:
Jaret Burkett committed 2026-08-29 11:23:47 -06:00
1 parent 64a20f51a6
commit 683fe8afc0
11 files changed
+390 -45

No files matched your search

+61 -4
View File
@@ -218,6 +218,11 @@ class OstrisModelMixin:
config=config,
subfolder=subfolder,
use_comfy_weights=use_comfy_weights,
# comfy candidate ranking prefers the file whose shipped
# quantization matches the request (strip any ARA suffix);
# quantization itself happens in aitk_post_load below
qtype=(qtype or "").split("|", 1)[0] or None,
quantize_on_load=False,
**kwargs,
)
return model.aitk_post_load(
@@ -254,6 +259,53 @@ class OstrisModelMixin:
if qtype is not None and "|" in qtype:
qtype, ara_path = qtype.split("|", 1)
# pre-quantized checkpoint: keep the shipped quantization IFF it
# exactly matches the request. Anything else — no quantization
# requested (full finetuning), a different backend, or an ARA —
# restores/requantizes to what was asked for.
if getattr(self, "aitk_is_quantized", False):
from toolkit.util.ostris_quant import OstrisLinear
from toolkit.util.quantize import (
dequantize_ostris_to_linear,
get_qtype,
ostristype,
)
shipped = sorted(
{
getattr(m.ostris_quantizer, "qtype", None)
for m in self.modules()
if isinstance(m, OstrisLinear)
}
- {None}
)
matches = (
qtype is not None
and ara_path is None
and shipped == [qtype]
)
if matches:
status_fn(f"Checkpoint is pre-quantized ({qtype}); keeping it")
else:
target_is_ostris = qtype is not None and isinstance(
get_qtype(qtype), ostristype
)
if qtype is None or ara_path is not None or not target_is_ostris:
# full precision requested, an ARA (quantizes fresh from
# full weights), or a quanto/torchao backend that cannot
# re-quantize an OstrisLinear: dequantize first
status_fn(
f"Dequantizing shipped {'/'.join(shipped)} weights "
f"-> {qtype or 'full precision'}"
)
dequantize_ostris_to_linear(self)
else:
# ostris -> ostris: quantize_module re-quantizes each
# layer in place (dequant -> requant, one layer at a time)
status_fn(f"Requantizing {'/'.join(shipped)} -> {qtype}")
self.aitk_is_quantized = False
self.aitk_qtype = None
if qtype and not getattr(self, "aitk_is_quantized", False):
from toolkit.util.quantize import (
attach_ara_and_quantize,
@@ -308,8 +360,6 @@ class OstrisModelMixin:
)
self.aitk_is_quantized = True
self.aitk_qtype = qtype
elif qtype and getattr(self, "aitk_is_quantized", False):
status_fn("Checkpoint is pre-quantized; skipping quantization")
if offload and offload > 0:
from toolkit.memory_management import MemoryManager
@@ -337,6 +387,7 @@ class OstrisModelMixin:
config=None,
subfolder: Optional[str] = None,
use_comfy_weights: bool = True,
quantize_on_load: bool = True,
**kwargs,
):
"""Load a model universally from a given name or path.
@@ -370,7 +421,9 @@ class OstrisModelMixin:
and not name_or_path.endswith(".safetensors")
and not os.path.exists(name_or_path)
):
comfy_path = cls.resolve_comfy_weights(name_or_path, subfolder=subfolder)
comfy_path = cls.resolve_comfy_weights(
name_or_path, subfolder=subfolder, qtype=qtype
)
if comfy_path is not None:
if config_path is None:
# the standard repo supplies the config for the comfy file
@@ -399,7 +452,9 @@ class OstrisModelMixin:
name_or_path, subfolder=subfolder, dtype=dtype, **kwargs
)
if qtype is not None:
# quantize_on_load=False: qtype was only a comfy-candidate ranking
# hint (the .load() path quantizes in aitk_post_load instead)
if qtype is not None and quantize_on_load:
model.quantize_(
qtype, device=quantize_device, exclude=exclude_quant_modules
)
@@ -589,6 +644,7 @@ class OstrisModelMixin:
local_only: bool = False,
hf_token: Optional[str] = None,
status_fn: Optional[callable] = None,
qtype: Optional[str] = None,
) -> Optional[str]:
"""The comfy-format weight file replacing a standard ``name_or_path``,
or None when this class has none registered for it. Best-ranked local
@@ -618,6 +674,7 @@ class OstrisModelMixin:
hf_token=hf_token,
status_fn=status_fn,
local_only=local_only,
qtype=qtype,
)
# ------------------------------------------------------------------