diff --git a/jobs/process/BaseSDTrainProcess.py b/jobs/process/BaseSDTrainProcess.py index beb3686..a9233d6 100644 --- a/jobs/process/BaseSDTrainProcess.py +++ b/jobs/process/BaseSDTrainProcess.py @@ -370,6 +370,7 @@ class BaseSDTrainProcess(BaseTrainProcess): if self.ema is not None: self.ema.train() + print_acc("") # add a line break def update_training_metadata(self): o_dict = OrderedDict({ diff --git a/toolkit/print.py b/toolkit/print.py index fbbc82e..6da6cf7 100644 --- a/toolkit/print.py +++ b/toolkit/print.py @@ -1,6 +1,5 @@ import sys import os -import time from toolkit.accelerator import get_accelerator @@ -9,76 +8,15 @@ def print_acc(*args, **kwargs): print(*args, **kwargs) -# Progress bars (tqdm etc.) refresh many times a second with \r / cursor-up -# rewrites. The terminal gets every refresh untouched, but writing them all to -# the log file makes it enormous, so transient refreshes are buffered and -# written at most once per interval — each new refresh of the same line(s) -# replaces the buffered one. Real content (anything with actual text and a -# newline) always writes through immediately, preceded by any buffered refresh -# to preserve stream order. -TRANSIENT_WRITE_INTERVAL = 1.0 -# Safety valve: an unterminated refresh stream writes through past this size. -MAX_TRANSIENT_BUFFER = 65536 - - class Logger: def __init__(self, terminal, log_file): self.terminal = terminal self.log = log_file - # Last completed refresh cycle (a new cycle replaces the previous one). - self._cycle = '' - # In-progress refresh: a lone \r line rewrite, or a multi-write - # cursor-movement cycle (nested bars) still being assembled. - self._tail = '' - self._tail_is_lone_refresh = False - self._last_transient_write = 0.0 def write(self, message): self.terminal.write(message) - self._write_log(message) - - def _write_log(self, message): - has_move = '\r' in message or '\x1b[A' in message - has_newline = '\n' in message - if has_move and not has_newline: - if message.startswith('\r') and '\x1b[A' not in message and (not self._tail or self._tail_is_lone_refresh): - # Single-line rewrite (plain tqdm bar) — replaces the previous one. - self._tail = message - self._tail_is_lone_refresh = True - else: - # Part of a multi-write cursor-movement cycle (nested bars). - self._tail += message - self._tail_is_lone_refresh = False - elif self._tail and has_newline and message.strip('\r\n') == '': - # Pure newline movement closes a multi-write cycle; the completed - # cycle replaces the previously buffered one. - self._cycle = self._tail + message - self._tail = '' - self._tail_is_lone_refresh = False - else: - # Real content — write any buffered refresh first to keep order. - self._flush_transient() - self.log.write(message) - self.log.flush() - self._last_transient_write = time.monotonic() - return - now = time.monotonic() - # Only flush between cycles (or on a lone \r rewrite, which leaves the - # cursor on the same row) — flushing mid-cycle would leave the file's - # cursor moved up and misalign everything written after. - mid_cycle = self._tail and not self._tail_is_lone_refresh - if (now - self._last_transient_write >= TRANSIENT_WRITE_INTERVAL and not mid_cycle) or len( - self._tail) > MAX_TRANSIENT_BUFFER: - self._flush_transient() - self._last_transient_write = now - - def _flush_transient(self): - if self._cycle or self._tail: - self.log.write(self._cycle + self._tail) - self.log.flush() - self._cycle = '' - self._tail = '' - self._tail_is_lone_refresh = False + self.log.write(message) + self.log.flush() # Make sure it's written immediately def flush(self): self.terminal.flush() diff --git a/version.py b/version.py index 3b779d2..7ab44ab 100644 --- a/version.py +++ b/version.py @@ -1 +1 @@ -VERSION = "0.10.26" +VERSION = "0.10.27"