From 9e9439015eb6dc05e8e8f4a082ba83f2e7fd43ee Mon Sep 17 00:00:00 2001
From: Jaret Burkett <jaretburkett@gmail.com>
Date: Tue, 21 Jul 2026 11:07:18 -0600
Subject: [PATCH] Switch sqlite database to WAL mode. It will be significantly
 faster with multiple writers and readers.

---
 .gitignore        |  2 ++
 ui/.gitignore     |  2 ++
 ui/cron/worker.ts | 51 ++++++++++++++++++++++++++++++++++++++++++++---
 3 files changed, 52 insertions(+), 3 deletions(-)

diff --git a/.gitignore b/.gitignore
index e6485fd..b045624 100644
--- a/.gitignore
+++ b/.gitignore
@@ -182,6 +182,8 @@ cython_debug/
 .DS_Store
 ._.DS_Store
 aitk_db.db
+aitk_db.db-wal
+aitk_db.db-shm
 /notes.md
 /data
 .claude
\ No newline at end of file
diff --git a/ui/.gitignore b/ui/.gitignore
index 421d02e..6c933de 100644
--- a/ui/.gitignore
+++ b/ui/.gitignore
@@ -40,3 +40,5 @@ yarn-error.log*
 *.tsbuildinfo
 next-env.d.ts
 aitk_db.db
+aitk_db.db-wal
+aitk_db.db-shm
diff --git a/ui/cron/worker.ts b/ui/cron/worker.ts
index dd1c275..053f0cb 100644
--- a/ui/cron/worker.ts
+++ b/ui/cron/worker.ts
@@ -1,4 +1,42 @@
 import processQueue from './actions/processQueue';
+import prisma from './prisma';
+
+// Journal mode for the main sqlite db. WAL keeps readers from blocking while
+// the trainer/worker write, which is what we want on a local disk. Users on
+// setups where WAL can't work (e.g. db on a network filesystem) can override
+// with AI_TOOLKIT_DB_JOURNAL_MODE=DELETE (or any other valid sqlite mode).
+const DEFAULT_JOURNAL_MODE = 'WAL';
+const VALID_JOURNAL_MODES = ['DELETE', 'TRUNCATE', 'PERSIST', 'MEMORY', 'WAL', 'OFF'];
+
+async function ensureJournalMode() {
+  const envMode = process.env.AI_TOOLKIT_DB_JOURNAL_MODE;
+  let targetMode = (envMode || DEFAULT_JOURNAL_MODE).toUpperCase();
+  if (!VALID_JOURNAL_MODES.includes(targetMode)) {
+    console.warn(
+      `Invalid AI_TOOLKIT_DB_JOURNAL_MODE "${envMode}", expected one of ${VALID_JOURNAL_MODES.join(', ')}. Using ${DEFAULT_JOURNAL_MODE}.`,
+    );
+    targetMode = DEFAULT_JOURNAL_MODE;
+  }
+
+  const current = await prisma.$queryRawUnsafe<{ journal_mode: string }[]>('PRAGMA journal_mode;');
+  const currentMode = current[0]?.journal_mode?.toUpperCase();
+  if (currentMode === targetMode) {
+    return;
+  }
+
+  console.log(`Converting database journal mode from ${currentMode} to ${targetMode}...`);
+  // targetMode is validated against VALID_JOURNAL_MODES above, safe to interpolate
+  const result = await prisma.$queryRawUnsafe<{ journal_mode: string }[]>(`PRAGMA journal_mode = ${targetMode};`);
+  const resultMode = result[0]?.journal_mode?.toUpperCase();
+  if (resultMode === targetMode) {
+    console.log(`Database journal mode is now ${resultMode}.`);
+  } else {
+    // sqlite refuses the switch rather than corrupting anything (e.g. WAL on a
+    // network filesystem), so just report what we're actually running with.
+    console.warn(`Could not convert database journal mode to ${targetMode}, still using ${resultMode}.`);
+  }
+}
+
 class CronWorker {
   interval: number;
   is_running: boolean;
@@ -29,6 +67,13 @@ class CronWorker {
   }
 }
 
-// it automatically starts the loop
-const cronWorker = new CronWorker();
-console.log('Cron worker started with interval:', cronWorker.interval, 'ms');
+// make sure the db journal mode is set before the loop starts hitting the db
+ensureJournalMode()
+  .catch(error => {
+    console.warn('Could not check/convert database journal mode:', error);
+  })
+  .finally(() => {
+    // it automatically starts the loop
+    const cronWorker = new CronWorker();
+    console.log('Cron worker started with interval:', cronWorker.interval, 'ms');
+  });