diff --git a/manager/README.md b/manager/README.md index a103a7b..3e856fb 100644 --- a/manager/README.md +++ b/manager/README.md @@ -46,6 +46,19 @@ python3 -m manager doctor # diagnose problems before the optional accelerators are import-tested and again at the end. Requirements files must never pin a torch dependency below what torch needs (torch 2.13 wants `setuptools>=77.0.3`). +- **`ui/package-lock.json` is never modified by installing.** A plain + `npm install` re-derives the lockfile for whichever machine runs it — on + Windows it strips the `libc` fields off the Linux-only optional binaries + (`@next/swc-linux-*`, rollup, lightningcss), on Linux it adds them back — so + with the install baked into `npm run build_and_start` every user got a dirty + tree on every launch, which then blocks `manager update` (a dirty tree aborts + the pull). `nodejs.ensure_ui_deps` owns the install instead: `npm install + --no-save`, gated on a hash of `ui/package.json` + `ui/package-lock.json` + (stored in the venv state), with the lockfile bytes snapshotted and restored + either way. `manager launch` therefore runs `npm run db_build_start`, not + `build_and_start`; the latter still exists for the manual + `cd ui && npm run build_and_start` flow in the README and calls the same + non-writing install via `npm run install_deps`. - **Nothing global is ever installed.** FFmpeg (shared builds — the libs torchcodec dlopens) goes to `.ffmpeg/` ([ffmpeg.py](ffmpeg.py)), Node (when the system lacks >= 20) to `.node/` ([nodejs.py](nodejs.py)), the uv diff --git a/manager/env.py b/manager/env.py index f376e49..cd0d380 100644 --- a/manager/env.py +++ b/manager/env.py @@ -544,6 +544,7 @@ def sync(spec, detection, dry_run=False, force=False): ensure_requirements(spec, dry_run=dry_run, force=force or changed_torch) ffmpeg.ensure_ffmpeg(detection, dry_run=dry_run) nodejs.ensure_node(detection, dry_run=dry_run) + nodejs.ensure_ui_deps(dry_run=dry_run) write_sitecustomize(dry_run=dry_run) migrations.run_pending(dry_run=dry_run) ok("Environment is up to date.") diff --git a/manager/launch.py b/manager/launch.py index 68ca4c6..d94aca8 100644 --- a/manager/launch.py +++ b/manager/launch.py @@ -1,4 +1,4 @@ -"""Launch the AI Toolkit web UI (ui/ -> npm run build_and_start).""" +"""Launch the AI Toolkit web UI (ui/ -> npm run db_build_start).""" import os import subprocess @@ -45,12 +45,6 @@ def build_env(): return env -def _find_npm(env): - import shutil - - return shutil.which("npm.cmd" if IS_WINDOWS else "npm", path=env.get("PATH")) - - def _headless(): return IS_LINUX and not ( os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY") @@ -79,7 +73,7 @@ def launch_ui(open_browser=True): die("No Python environment found. Run: python3 -m manager install") env = build_env() - npm = _find_npm(env) + npm = nodejs.find_npm(env) if not npm: die( "Node.js was not found. Run `python3 -m manager sync` to install a " @@ -93,6 +87,10 @@ def launch_ui(open_browser=True): % (major, nodejs.MIN_NODE_MAJOR) ) + # deps are installed here rather than by the npm script so the lockfile is + # never rewritten (see nodejs.ensure_ui_deps); a no-op once they're in sync + nodejs.ensure_ui_deps(env=env) + info("Starting AI Toolkit UI (%s) ..." % UI_URL) stop_event = threading.Event() if open_browser and not _headless(): @@ -101,7 +99,7 @@ def launch_ui(open_browser=True): ).start() proc = subprocess.Popen( - [npm, "run", "build_and_start"], cwd=os.path.join(REPO_ROOT, "ui"), env=env + [npm, "run", "db_build_start"], cwd=os.path.join(REPO_ROOT, "ui"), env=env ) try: return proc.wait() diff --git a/manager/nodejs.py b/manager/nodejs.py index 6eb8b93..011f1e9 100644 --- a/manager/nodejs.py +++ b/manager/nodejs.py @@ -14,10 +14,13 @@ from .util import ( IS_MAC, IS_WINDOWS, REPO_ROOT, + clean_env, download, extract_archive, + file_hash, info, ok, + run, warn, which, ) @@ -27,6 +30,9 @@ NODE_DIR = os.path.join(REPO_ROOT, ".node") NODE_VERSION = "24.11.1" MIN_NODE_MAJOR = 20 +UI_DIR = os.path.join(REPO_ROOT, "ui") +UI_STATE_KEY = "ui_deps_hash" + def node_bin_dir(): # windows zips have node.exe/npm.cmd at the archive root; unix under bin/ @@ -119,3 +125,93 @@ def ensure_node(detection, dry_run=False): return True finally: shutil.rmtree(tmp, ignore_errors=True) + + +# ---------------------------------------------------------------- ui deps + + +def npm_env(): + """Scrubbed env with the local .node/ ahead of PATH.""" + env = clean_env() + if os.path.isdir(node_bin_dir()): + env["PATH"] = node_bin_dir() + os.pathsep + env.get("PATH", "") + return env + + +def find_npm(env=None): + """npm from the local .node/ copy when we installed one, else the system.""" + path = (env or npm_env()).get("PATH") + return shutil.which("npm.cmd" if IS_WINDOWS else "npm", path=path) + + +def _ui_deps_hash(): + return file_hash( + [ + os.path.join(UI_DIR, "package-lock.json"), + os.path.join(UI_DIR, "package.json"), + ] + ) + + +def ensure_ui_deps(env=None, dry_run=False): + """Install ui/node_modules without ever rewriting ui/package-lock.json. + + A plain `npm install` treats the lockfile as writable and re-derives it for + the machine doing the install: on Windows it strips the `libc` fields off + the Linux-only optional binaries (@next/swc-linux-*, rollup, lightningcss), + on Linux it puts them back. Since the UI installs on every launch, everyone + ends up with a modified lockfile they never asked for — which then blocks + `manager update`, because a dirty tree aborts the pull. + + `--no-save` keeps the resolution but stops the write-back, and stays an + incremental install (seconds) rather than the full wipe-and-refetch of + `npm ci`. The lockfile is snapshotted and restored regardless, so this + holds even if a future npm changes what `--no-save` covers. + + Gated on a hash of the manifests so steady-state launches skip npm. + """ + from . import env as env_mod + + lock = os.path.join(UI_DIR, "package-lock.json") + if not os.path.isfile(lock): + warn("ui/package-lock.json is missing — skipping UI dependency install.") + return False + want = _ui_deps_hash() + if env_mod.venv_exists() and env_mod.load_state().get(UI_STATE_KEY) == want: + if os.path.isdir(os.path.join(UI_DIR, "node_modules")): + ok("UI dependencies already installed.") + return False + if dry_run: + info("[dry-run] would run: npm install --no-save (in %s)" % UI_DIR) + return False + env = env or npm_env() + npm = find_npm(env) + if npm is None: + warn("npm not found — skipping UI dependency install.") + return False + + info("Installing UI dependencies...") + with open(lock, "rb") as f: + before = f.read() + code, _ = run( + [npm, "install", "--no-save", "--no-audit", "--no-fund"], + cwd=UI_DIR, + env=env, + stream=True, + check=False, + ) + with open(lock, "rb") as f: + after = f.read() + if after != before: + with open(lock, "wb") as f: + f.write(before) + info("Reverted npm's edit to ui/package-lock.json (managed by the repo).") + if code != 0: + warn("UI dependency install failed — the UI may not start.") + return False + if env_mod.venv_exists(): + state = env_mod.load_state() + state[UI_STATE_KEY] = want + env_mod.save_state(state) + ok("UI dependencies ready.") + return True diff --git a/ui/package.json b/ui/package.json index b799bba..0b72a0b 100644 --- a/ui/package.json +++ b/ui/package.json @@ -6,7 +6,9 @@ "dev": "concurrently -k -n WORKER,UI \"ts-node-dev --project tsconfig.worker.json --respawn --watch cron --transpile-only cron/worker.ts\" \"ts-node-dev --project tsconfig.worker.json --respawn --watch cron --transpile-only cron/fileServer.ts dev --port 3000\"", "build": "tsc -p tsconfig.worker.json && next build", "start": "concurrently --restart-tries -1 --restart-after 1000 -n WORKER,UI \"node dist/cron/worker.js\" \"node dist/cron/fileServer.js start --port 8675\"", - "build_and_start": "npm install && npm run update_db && npm run build && npm run start", + "install_deps": "npm install --no-save --no-audit --no-fund", + "db_build_start": "npm run update_db && npm run build && npm run start", + "build_and_start": "npm run install_deps && npm run db_build_start", "lint": "next lint", "update_db": "npx prisma generate && npx prisma db push", "format": "prettier --write \"**/*.{js,jsx,ts,tsx,css,scss}\""