github/ai-toolkit
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Generate thumbnails for dataset items so things load faster and the ui is more stable. Especially good for videos.

Browse Source
This commit is contained in:
Jaret Burkett
2026-08-12 20:33:42 -06:00
parent 175cc1e151
commit 7f9a142dfd
4 changed files with 185 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
toolkit/data_loader.py
View File

@@ -434,7 +434,11 @@ class AiToolkitDataset(LatentCachingMixin, ControlCachingMixin, CLIPCachingMixin
# look for videos and images. Video models can train on both;
# images are bucketed separately as single-frame items
extensions = video_extensions + image_extensions
file_list = [os.path.join(root, file) for root, _, files in os.walk(self.dataset_path) for file in files if file.lower().endswith(tuple(extensions)) and not file.startswith('.')]
# prune hidden dirs (.thumbs, .tmp) so their contents never train
file_list = []
for root, dirs, files in os.walk(self.dataset_path):
dirs[:] = [d for d in dirs if not d.startswith('.')]
file_list.extend(os.path.join(root, file) for file in files if file.lower().endswith(tuple(extensions)) and not file.startswith('.'))
else:
# assume json
with open(self.dataset_path, 'r') as f: