From 93a0f2e7542d04aa98d57a8fd22ddf3f07c475e5 Mon Sep 17 00:00:00 2001 From: ROHAN PANDEY <95585299+rohan-pandeyy@users.noreply.github.com> Date: Fri, 24 Jul 2026 11:19:43 +0530 Subject: [PATCH 01/10] feat(backend): video keyframe sampling and tagging pipeline Extend AI tagging to videos without per-frame inference: sample one keyframe every N seconds (capped per video, interval stretched for long videos), write them as JPEGs, and run them through the existing image taggers. New video_frames / video_frame_embeddings / video_classes tables mirror the image side and are kept separate so frames never leak into the photo gallery, memories, or face clustering. YOLO object tags are aggregated across frames by frame support; SigLIP2 frame embeddings are scored per-video against the curated vocabulary (best frame wins). A shared scoring-context helper is factored out of the image pass and reused. The tagging, embedding, and scoring passes run after the image passes in the enable-tagging and sync sequences. --- backend/app/config/settings.py | 20 + backend/app/database/video_frames.py | 524 +++++++++++++++++++++++++++ backend/app/database/videos.py | 110 ++++-- backend/app/routes/folders.py | 18 +- backend/app/utils/semantic_labels.py | 187 ++++++++-- backend/app/utils/videos.py | 349 +++++++++++++++++- backend/main.py | 4 + backend/tests/conftest.py | 2 + 8 files changed, 1146 insertions(+), 68 deletions(-) create mode 100644 backend/app/database/video_frames.py diff --git a/backend/app/config/settings.py b/backend/app/config/settings.py index bcb80cd3a..bc3204102 100644 --- a/backend/app/config/settings.py +++ b/backend/app/config/settings.py @@ -39,6 +39,7 @@ else: DATABASE_PATH = os.path.join(user_data_dir("PictoPy"), "database", "PictoPy.db") THUMBNAIL_IMAGES_PATH = os.path.join(user_data_dir("PictoPy"), "thumbnails") +VIDEO_FRAMES_PATH = os.path.join(user_data_dir("PictoPy"), "video_frames") IMAGES_PATH = "./images" @@ -175,6 +176,25 @@ def _get_env_int( } SEMANTIC_DEFAULT_THRESHOLD = 5e-05 +# Video keyframe sampling. One frame every N seconds instead of per-frame +# inference (30fps = 1800 forward passes per minute of video). The interval is +# stretched when a video would exceed the cap, so cost per video is bounded. +VIDEO_FRAME_INTERVAL_SECONDS = _get_env_float( + "VIDEO_FRAME_INTERVAL_SECONDS", 5.0, min_value=0.5, max_value=300.0 +) +VIDEO_MAX_FRAMES_PER_VIDEO = _get_env_int( + "VIDEO_MAX_FRAMES_PER_VIDEO", 200, min_value=1 +) +# Frames are saved above SigLIP2's largest input (384) so a checkpoint swap +# doesn't require re-extraction. +VIDEO_FRAME_MAX_DIMENSION = _get_env_int("VIDEO_FRAME_MAX_DIMENSION", 640, min_value=64) +# A tag must appear in this many frames to describe the video; drops one-off +# detections from a single unlucky keyframe. +VIDEO_TAG_MIN_FRAME_SUPPORT = _get_env_int( + "VIDEO_TAG_MIN_FRAME_SUPPORT", 2, min_value=1 +) +VIDEO_TAG_TOP_K = _get_env_int("VIDEO_TAG_TOP_K", 15, min_value=1) + # Clustering Configuration PICTO_CLUSTERING_EPS = _get_env_float("PICTO_CLUSTERING_EPS", 0.75, min_value=0.0) PICTO_CLUSTERING_MIN_SAMPLES = _get_env_int( diff --git a/backend/app/database/video_frames.py b/backend/app/database/video_frames.py new file mode 100644 index 000000000..98a9c1fe2 --- /dev/null +++ b/backend/app/database/video_frames.py @@ -0,0 +1,524 @@ +"""Sampled video keyframes and the tags derived from them. + +Frames are deliberately kept out of the images table: they would otherwise +leak into the photo gallery, memories and face clustering. These tables +mirror their image counterparts so the existing tagging passes port over +with the same shapes. +""" + +import sqlite3 +from typing import Dict, List, Optional, Tuple + +import numpy as np + +from app.database.images import _connect +from app.database.semantic_labels import SEMANTIC_CLASS_ID_OFFSET +from app.logging.setup_logging import get_logger + +logger = get_logger(__name__) + + +def db_create_video_frames_tables() -> None: + conn = None + try: + conn = _connect() + cursor = conn.cursor() + + # frame_path is nullable so the purge action can drop the JPEGs + # without destroying the embeddings that depend on them. + cursor.execute( + """ + CREATE TABLE IF NOT EXISTS video_frames ( + id TEXT PRIMARY KEY, + video_id TEXT NOT NULL, + frame_path TEXT UNIQUE, + timestamp_sec REAL, + frame_index INTEGER, + isEmbedded BOOLEAN DEFAULT 0, + FOREIGN KEY (video_id) REFERENCES videos(id) ON DELETE CASCADE + ) + """ + ) + cursor.execute( + "CREATE INDEX IF NOT EXISTS ix_video_frames_video_id " + "ON video_frames(video_id)" + ) + + # Same raw-float32 blob format as image_embeddings.embedding. + cursor.execute( + """ + CREATE TABLE IF NOT EXISTS video_frame_embeddings ( + frame_id TEXT PRIMARY KEY, + model_version TEXT NOT NULL, + embedding BLOB NOT NULL, + scored_signature TEXT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (frame_id) REFERENCES video_frames(id) ON DELETE CASCADE + ) + """ + ) + cursor.execute( + "CREATE INDEX IF NOT EXISTS ix_video_frame_embeddings_model_version " + "ON video_frame_embeddings(model_version)" + ) + + # Video-level tags, aggregated from the frames at write time so tag + # queries stay a plain join. score is NULL for YOLO rows and the best + # frame's match score for semantic rows, matching image_classes. + cursor.execute( + """ + CREATE TABLE IF NOT EXISTS video_classes ( + video_id TEXT, + class_id INTEGER, + score REAL, + frame_count INTEGER, + PRIMARY KEY (video_id, class_id), + FOREIGN KEY (video_id) REFERENCES videos(id) ON DELETE CASCADE, + FOREIGN KEY (class_id) REFERENCES mappings(class_id) ON DELETE CASCADE + ) + """ + ) + + # Display cut mirroring image_classes_display: all YOLO tags, but only + # the top-SEMANTIC_DISPLAY_TOP_K semantic tags per video. Recreated at + # startup so setting changes apply without re-scoring. + from app.config.settings import SEMANTIC_DISPLAY_TOP_K + + cursor.execute("DROP VIEW IF EXISTS video_classes_display") + cursor.execute( + f""" + CREATE VIEW video_classes_display AS + SELECT video_id, class_id FROM ( + SELECT video_id, class_id, + ROW_NUMBER() OVER ( + PARTITION BY video_id ORDER BY score DESC + ) AS display_rank + FROM video_classes WHERE score IS NOT NULL + ) WHERE display_rank <= {int(SEMANTIC_DISPLAY_TOP_K)} + UNION ALL + SELECT video_id, class_id FROM video_classes WHERE score IS NULL + """ + ) + + conn.commit() + finally: + if conn: + conn.close() + + +def db_get_untagged_videos() -> List[Dict[str, str]]: + """Videos in AI-tagging-enabled folders that have not been tagged yet.""" + conn = None + try: + conn = _connect() + cursor = conn.cursor() + cursor.execute( + """ + SELECT v.id, v.path + FROM videos v + JOIN folders f ON v.folder_id = f.folder_id + WHERE f.AI_Tagging = TRUE + AND v.isTagged = FALSE + """ + ) + return [{"id": video_id, "path": path} for video_id, path in cursor.fetchall()] + finally: + if conn: + conn.close() + + +def db_mark_videos_tagged(video_ids: List[str]) -> bool: + if not video_ids: + return True + + conn = None + try: + conn = _connect() + cursor = conn.cursor() + placeholders = ",".join("?" for _ in video_ids) + cursor.execute( + f"UPDATE videos SET isTagged = 1 WHERE id IN ({placeholders})", + video_ids, + ) + conn.commit() + return True + except sqlite3.Error as e: + logger.error(f"Error marking videos as tagged: {e}") + if conn: + conn.rollback() + return False + finally: + if conn: + conn.close() + + +def db_bulk_insert_video_frames(frame_records: List[dict]) -> bool: + """Insert sampled frame rows: id, video_id, frame_path, timestamp_sec, + frame_index.""" + if not frame_records: + return True + + conn = None + try: + conn = _connect() + cursor = conn.cursor() + cursor.executemany( + """ + INSERT INTO video_frames + (id, video_id, frame_path, timestamp_sec, frame_index) + VALUES (:id, :video_id, :frame_path, :timestamp_sec, :frame_index) + ON CONFLICT(frame_path) DO UPDATE SET + video_id=excluded.video_id, + timestamp_sec=excluded.timestamp_sec, + frame_index=excluded.frame_index + """, + frame_records, + ) + conn.commit() + return True + except sqlite3.Error as e: + logger.error(f"Error inserting video frames: {e}") + if conn: + conn.rollback() + return False + finally: + if conn: + conn.close() + + +def db_delete_frames_for_videos(video_ids: List[str]) -> bool: + """Drop a video's frame rows (and, by cascade, their embeddings) so a + re-tag starts from a clean sample.""" + if not video_ids: + return True + + conn = None + try: + conn = _connect() + cursor = conn.cursor() + placeholders = ",".join("?" for _ in video_ids) + cursor.execute( + f"DELETE FROM video_frames WHERE video_id IN ({placeholders})", + video_ids, + ) + conn.commit() + return True + except sqlite3.Error as e: + logger.error(f"Error deleting video frames: {e}") + if conn: + conn.rollback() + return False + finally: + if conn: + conn.close() + + +def db_get_unembedded_video_frames() -> List[Dict[str, str]]: + """Frames still needing a SigLIP2 embedding. Purged frames (frame_path + NULL) are excluded -- there is nothing left on disk to embed.""" + conn = None + try: + conn = _connect() + cursor = conn.cursor() + cursor.execute( + """ + SELECT vf.id, vf.frame_path + FROM video_frames vf + JOIN videos v ON vf.video_id = v.id + JOIN folders f ON v.folder_id = f.folder_id + WHERE f.AI_Tagging = TRUE + AND vf.isEmbedded = FALSE + AND vf.frame_path IS NOT NULL + """ + ) + return [ + {"id": frame_id, "frame_path": frame_path} + for frame_id, frame_path in cursor.fetchall() + ] + finally: + if conn: + conn.close() + + +def db_mark_video_frames_embedded(frame_ids: List[str]) -> bool: + if not frame_ids: + return True + + conn = None + try: + conn = _connect() + cursor = conn.cursor() + # Chunked by 500 to stay under SQLite's 999-variable limit, matching + # db_mark_images_embedded. + chunk_size = 500 + for i in range(0, len(frame_ids), chunk_size): + chunk = frame_ids[i : i + chunk_size] + placeholders = ",".join("?" for _ in chunk) + cursor.execute( + f"UPDATE video_frames SET isEmbedded = 1 WHERE id IN ({placeholders})", + chunk, + ) + conn.commit() + return True + except sqlite3.Error as e: + logger.error(f"Error marking video frames as embedded: {e}") + if conn: + conn.rollback() + return False + finally: + if conn: + conn.close() + + +def db_upsert_video_frame_embeddings( + rows: List[Tuple[str, str, np.ndarray]], +) -> None: + conn = None + try: + conn = _connect() + cursor = conn.cursor() + cursor.executemany( + """ + INSERT INTO video_frame_embeddings (frame_id, model_version, embedding) + VALUES (?, ?, ?) + ON CONFLICT(frame_id) DO UPDATE SET + model_version = excluded.model_version, + embedding = excluded.embedding, + scored_signature = NULL, + created_at = CURRENT_TIMESTAMP + """, + [ + ( + frame_id, + model_version, + np.ascontiguousarray(embedding, dtype=np.float32).tobytes(), + ) + for frame_id, model_version, embedding in rows + ], + ) + conn.commit() + finally: + if conn: + conn.close() + + +def db_write_video_classes(video_id: str, pairs: List[Tuple[int, int]]) -> None: + """Replace a video's YOLO tag rows with (class_id, frame_count) pairs. + Semantic rows (class_id at or above the offset) are never touched.""" + conn = None + try: + conn = _connect() + cursor = conn.cursor() + cursor.execute( + "DELETE FROM video_classes WHERE video_id = ? AND class_id < ?", + (video_id, SEMANTIC_CLASS_ID_OFFSET), + ) + cursor.executemany( + "INSERT OR REPLACE INTO video_classes " + "(video_id, class_id, score, frame_count) VALUES (?, ?, NULL, ?)", + [(video_id, class_id, frame_count) for class_id, frame_count in pairs], + ) + conn.commit() + finally: + if conn: + conn.close() + + +def db_write_video_semantic_scores( + video_id: str, pairs: List[Tuple[int, float, int]], signature: str +) -> None: + """Replace a video's semantic tag rows with (class_id, score, frame_count) + triples and stamp the signature on every frame of that video. YOLO rows + are never touched -- mirrors db_write_image_semantic_scores.""" + conn = None + try: + conn = _connect() + cursor = conn.cursor() + cursor.execute( + "DELETE FROM video_classes WHERE video_id = ? AND class_id >= ?", + (video_id, SEMANTIC_CLASS_ID_OFFSET), + ) + cursor.executemany( + "INSERT OR REPLACE INTO video_classes " + "(video_id, class_id, score, frame_count) VALUES (?, ?, ?, ?)", + [ + (video_id, class_id, score, frame_count) + for class_id, score, frame_count in pairs + ], + ) + cursor.execute( + """ + UPDATE video_frame_embeddings SET scored_signature = ? + WHERE frame_id IN (SELECT id FROM video_frames WHERE video_id = ?) + """, + (signature, video_id), + ) + conn.commit() + finally: + if conn: + conn.close() + + +def db_get_videos_needing_scoring( + model_version: str, signature: str, limit: int +) -> List[str]: + """Videos with at least one frame embedding whose semantic scores are + missing or computed against another vocabulary/label state.""" + conn = None + try: + conn = _connect() + cursor = conn.cursor() + cursor.execute( + """ + SELECT DISTINCT vf.video_id + FROM video_frames vf + JOIN video_frame_embeddings e ON vf.id = e.frame_id + WHERE e.model_version = ? + AND IFNULL(e.scored_signature, '') != ? + LIMIT ? + """, + (model_version, signature, limit), + ) + return [row[0] for row in cursor.fetchall()] + finally: + if conn: + conn.close() + + +def db_get_frame_embeddings_for_video(video_id: str, model_version: str) -> np.ndarray: + """All of one video's frame embeddings as an [N, D] matrix.""" + conn = None + try: + conn = _connect() + cursor = conn.cursor() + cursor.execute( + """ + SELECT e.embedding + FROM video_frames vf + JOIN video_frame_embeddings e ON vf.id = e.frame_id + WHERE vf.video_id = ? AND e.model_version = ? + ORDER BY vf.frame_index + """, + (video_id, model_version), + ) + rows = cursor.fetchall() + if not rows: + return np.empty((0, 0), dtype=np.float32) + return np.vstack([np.frombuffer(blob, dtype=np.float32) for (blob,) in rows]) + finally: + if conn: + conn.close() + + +def db_get_all_frame_embeddings( + model_version: str, +) -> Tuple[List[str], List[float], np.ndarray]: + """Every frame embedding with its video and timestamp, for semantic + search. Returns (video_ids, timestamps, matrix) aligned by row.""" + conn = None + try: + conn = _connect() + cursor = conn.cursor() + cursor.execute( + """ + SELECT vf.video_id, vf.timestamp_sec, e.embedding + FROM video_frames vf + JOIN video_frame_embeddings e ON vf.id = e.frame_id + WHERE e.model_version = ? + """, + (model_version,), + ) + rows = cursor.fetchall() + if not rows: + return [], [], np.empty((0, 0), dtype=np.float32) + + video_ids = [row[0] for row in rows] + timestamps = [row[1] for row in rows] + matrix = np.vstack([np.frombuffer(row[2], dtype=np.float32) for row in rows]) + return video_ids, timestamps, matrix + finally: + if conn: + conn.close() + + +def db_get_video_tags(video_ids: Optional[List[str]] = None) -> Dict[str, List[str]]: + """Display tag names per video. Pass video_ids to restrict, or None for + every video.""" + conn = None + try: + conn = _connect() + cursor = conn.cursor() + + query = """ + SELECT vc.video_id, m.name + FROM video_classes_display vc + JOIN mappings m ON vc.class_id = m.class_id + """ + params: List[str] = [] + if video_ids is not None: + if not video_ids: + return {} + placeholders = ",".join("?" for _ in video_ids) + query += f" WHERE vc.video_id IN ({placeholders})" + params = video_ids + query += " ORDER BY vc.video_id, m.name" + + cursor.execute(query, params) + + tags: Dict[str, List[str]] = {} + for video_id, name in cursor.fetchall(): + tags.setdefault(video_id, []).append(name) + return tags + except sqlite3.Error as e: + logger.error(f"Error getting video tags: {e}") + return {} + finally: + if conn: + conn.close() + + +def db_get_video_ids_by_tag(tag_name: str) -> List[str]: + """Video IDs carrying a tag. Matches against the full stored tag set, not + the display cut, so search isn't limited by SEMANTIC_DISPLAY_TOP_K.""" + conn = None + try: + conn = _connect() + cursor = conn.cursor() + cursor.execute( + """ + SELECT DISTINCT vc.video_id + FROM video_classes vc + JOIN mappings m ON vc.class_id = m.class_id + WHERE LOWER(m.name) = LOWER(?) + """, + (tag_name,), + ) + return [row[0] for row in cursor.fetchall()] + except sqlite3.Error as e: + logger.error(f"Error searching videos by tag: {e}") + raise + finally: + if conn: + conn.close() + + +def db_clear_frame_paths() -> bool: + """Forget where the frame JPEGs were, keeping the rows and their + embeddings so tags and semantic search survive a purge.""" + conn = None + try: + conn = _connect() + cursor = conn.cursor() + cursor.execute( + "UPDATE video_frames SET frame_path = NULL WHERE frame_path IS NOT NULL" + ) + conn.commit() + return True + except sqlite3.Error as e: + logger.error(f"Error clearing video frame paths: {e}") + if conn: + conn.rollback() + return False + finally: + if conn: + conn.close() diff --git a/backend/app/database/videos.py b/backend/app/database/videos.py index e9e528334..f8c875eed 100644 --- a/backend/app/database/videos.py +++ b/backend/app/database/videos.py @@ -44,9 +44,10 @@ def db_create_videos_table() -> None: conn = _connect() cursor = conn.cursor() - # Videos are kept separate from images by design; isTagged is reserved - # for future AI video tagging. thumbnailPath is nullable: videos whose - # codec OpenCV cannot decode are still indexed (frontend shows a placeholder). + # Videos are kept separate from images by design; isTagged tracks the + # keyframe sampling pass (see video_frames). thumbnailPath is nullable: + # videos whose codec OpenCV cannot decode are still indexed (frontend + # shows a placeholder). cursor.execute( """ CREATE TABLE IF NOT EXISTS videos ( @@ -127,34 +128,7 @@ def db_get_all_videos() -> List[dict]: results = cursor.fetchall() - videos = [] - for ( - video_id, - path, - folder_id, - thumbnail_path, - metadata, - is_tagged, - is_favourite, - captured_at, - ) in results: - from app.utils.images import image_util_parse_metadata - - videos.append( - { - "id": video_id, - "path": path, - "folder_id": str(folder_id), - "thumbnailPath": thumbnail_path, - "metadata": image_util_parse_metadata(metadata), - "isTagged": bool(is_tagged), - "isFavourite": bool(is_favourite), - "captured_at": captured_at if captured_at else None, - "tags": None, - } - ) - - return videos + return _build_video_dicts(results) except sqlite3.Error as e: logger.error(f"Error getting all videos: {e}") @@ -163,6 +137,42 @@ def db_get_all_videos() -> List[dict]: conn.close() +def _build_video_dicts(rows: List[Tuple]) -> List[dict]: + """Turn the shared 8-column video SELECT into dicts, attaching each + video's display tags.""" + from app.database.video_frames import db_get_video_tags + from app.utils.images import image_util_parse_metadata + + tags_by_video = db_get_video_tags([row[0] for row in rows]) + + videos = [] + for ( + video_id, + path, + folder_id, + thumbnail_path, + metadata, + is_tagged, + is_favourite, + captured_at, + ) in rows: + videos.append( + { + "id": video_id, + "path": path, + "folder_id": str(folder_id), + "thumbnailPath": thumbnail_path, + "metadata": image_util_parse_metadata(metadata), + "isTagged": bool(is_tagged), + "isFavourite": bool(is_favourite), + "captured_at": captured_at if captured_at else None, + "tags": tags_by_video.get(video_id) or None, + } + ) + + return videos + + def db_get_video_index_by_folder_ids( folder_ids: List[int], ) -> List[Tuple[VideoPath, Optional[str], Optional[str]]]: @@ -200,6 +210,44 @@ def db_get_video_index_by_folder_ids( conn.close() +def db_get_videos_by_ids(video_ids: List[VideoId]) -> List[dict]: + """Get videos by ID, preserving the order of video_ids.""" + if not video_ids: + return [] + + conn = _connect() + cursor = conn.cursor() + + try: + rows = [] + # Chunked by 500 to stay under SQLite's 999-variable limit. + chunk_size = 500 + for i in range(0, len(video_ids), chunk_size): + chunk = video_ids[i : i + chunk_size] + placeholders = ",".join("?" for _ in chunk) + cursor.execute( + f""" + SELECT id, path, folder_id, thumbnailPath, metadata, isTagged, + isFavourite, captured_at + FROM videos + WHERE id IN ({placeholders}) + """, + chunk, + ) + rows.extend(cursor.fetchall()) + + video_lookup = {video["id"]: video for video in _build_video_dicts(rows)} + return [ + video_lookup[video_id] for video_id in video_ids if video_id in video_lookup + ] + + except sqlite3.Error as e: + logger.error(f"Error getting videos by IDs: {e}") + raise + finally: + conn.close() + + def db_get_videos_by_folder_ids( folder_ids: List[int], ) -> List[Tuple[VideoId, VideoPath, str]]: diff --git a/backend/app/routes/folders.py b/backend/app/routes/folders.py index d6e1005cf..537ea262c 100644 --- a/backend/app/routes/folders.py +++ b/backend/app/routes/folders.py @@ -43,9 +43,16 @@ image_util_process_untagged_images, image_util_process_unembedded_images, ) -from app.utils.videos import video_util_process_folder_videos +from app.utils.videos import ( + video_util_process_folder_videos, + video_util_process_untagged_videos, + video_util_process_unembedded_frames, +) from app.utils.model_bootstrap import ensure_ai_tagging_models -from app.utils.semantic_labels import semantic_util_score_images +from app.utils.semantic_labels import ( + semantic_util_score_images, + semantic_util_score_videos, +) from app.utils.face_clusters import cluster_util_face_clusters_sync from app.utils.API import API_util_restart_sync_microservice_watcher @@ -98,6 +105,10 @@ def post_AI_tagging_enabled_sequence(): cluster_util_face_clusters_sync() image_util_process_unembedded_images() semantic_util_score_images() + # Videos last: photos are the primary surface, so they finish first. + video_util_process_untagged_videos() + video_util_process_unembedded_frames() + semantic_util_score_videos() except Exception as e: logger.error(f"Error in post processing after AI tagging was enabled: {e}") return False @@ -129,6 +140,9 @@ def post_sync_folder_sequence( cluster_util_face_clusters_sync() image_util_process_unembedded_images() semantic_util_score_images() + video_util_process_untagged_videos() + video_util_process_unembedded_frames() + semantic_util_score_videos() # Restart sync microservice watcher after processing images API_util_restart_sync_microservice_watcher() diff --git a/backend/app/utils/semantic_labels.py b/backend/app/utils/semantic_labels.py index 2963fcfff..d7f4a174d 100644 --- a/backend/app/utils/semantic_labels.py +++ b/backend/app/utils/semantic_labels.py @@ -128,6 +128,57 @@ def semantic_util_build_label_embeddings() -> None: logger.error(f"Error building semantic label embeddings: {e}") +def _load_label_scoring_context(): + """Everything a scoring pass needs: the cached label matrix, per-label + thresholds, the signature that invalidates stale scores, and the + checkpoint's sigmoid calibration. Returns None when no label embeddings + are cached for the active checkpoint. + """ + import numpy as np + from app.config.settings import ( + SIGLIP2_ACTIVE_CHECKPOINT, + SIGLIP2_SCORING_METADATA, + SEMANTIC_SCORE_TOP_K, + SEMANTIC_BUCKET_THRESHOLDS, + SEMANTIC_DEFAULT_THRESHOLD, + ) + from app.database.semantic_labels import db_get_active_label_embeddings + + metadata = SIGLIP2_SCORING_METADATA[SIGLIP2_ACTIVE_CHECKPOINT] + model_version = metadata["model_version"] + + meta, label_matrix = db_get_active_label_embeddings(model_version) + if not meta: + return None + + thresholds = np.array( + [ + ( + override + if override is not None + else SEMANTIC_BUCKET_THRESHOLDS.get( + category, SEMANTIC_DEFAULT_THRESHOLD + ) + ) + for _, category, override in meta + ], + dtype=np.float32, + ) + signature = _scoring_signature( + model_version, SEMANTIC_SCORE_TOP_K, thresholds, meta, label_matrix + ) + + return { + "model_version": model_version, + "meta": meta, + "label_matrix": label_matrix, + "thresholds": thresholds, + "signature": signature, + "logit_scale": np.exp(metadata["logit_scale"]), + "logit_bias": metadata["logit_bias"], + } + + def semantic_util_score_images() -> None: """Score embedded images against the cached label matrix and write top-K above-threshold tags as image_classes rows. @@ -139,49 +190,26 @@ def semantic_util_score_images() -> None: """ import time import numpy as np - from app.config.settings import ( - SIGLIP2_ACTIVE_CHECKPOINT, - SIGLIP2_SCORING_METADATA, - SEMANTIC_SCORE_TOP_K, - SEMANTIC_BUCKET_THRESHOLDS, - SEMANTIC_DEFAULT_THRESHOLD, - ) - from app.database.semantic_labels import ( - db_get_active_label_embeddings, - db_write_image_semantic_scores, - ) + from app.config.settings import SEMANTIC_SCORE_TOP_K + from app.database.semantic_labels import db_write_image_semantic_scores from app.database.image_embeddings import db_get_embeddings_needing_scoring try: - metadata = SIGLIP2_SCORING_METADATA[SIGLIP2_ACTIVE_CHECKPOINT] - model_version = metadata["model_version"] - logit_scale = np.exp(metadata["logit_scale"]) - logit_bias = metadata["logit_bias"] - - meta, label_matrix = db_get_active_label_embeddings(model_version) - if not meta: + context = _load_label_scoring_context() + if context is None: logger.info( "No cached label embeddings for the active checkpoint; " "skipping semantic scoring pass" ) return - thresholds = np.array( - [ - ( - override - if override is not None - else SEMANTIC_BUCKET_THRESHOLDS.get( - category, SEMANTIC_DEFAULT_THRESHOLD - ) - ) - for _, category, override in meta - ], - dtype=np.float32, - ) - signature = _scoring_signature( - model_version, SEMANTIC_SCORE_TOP_K, thresholds, meta, label_matrix - ) + model_version = context["model_version"] + meta = context["meta"] + label_matrix = context["label_matrix"] + thresholds = context["thresholds"] + signature = context["signature"] + logit_scale = context["logit_scale"] + logit_bias = context["logit_bias"] total_images = 0 start_time = time.time() @@ -220,3 +248,94 @@ def semantic_util_score_images() -> None: ) except Exception as e: logger.error(f"Error in semantic scoring pass: {e}") + + +def semantic_util_score_videos() -> None: + """Score sampled keyframes and roll them up into video-level tags. + + Loops per video rather than in flat chunks like the image pass: a video's + tags are an aggregate over all its frames, so they have to be scored + together. A label's video score is its best frame's score, and it only + counts if enough frames agreed. + """ + import time + import numpy as np + from app.config.settings import VIDEO_TAG_MIN_FRAME_SUPPORT, VIDEO_TAG_TOP_K + from app.database.video_frames import ( + db_get_frame_embeddings_for_video, + db_get_videos_needing_scoring, + db_write_video_semantic_scores, + ) + + try: + context = _load_label_scoring_context() + if context is None: + logger.info( + "No cached label embeddings for the active checkpoint; " + "skipping video scoring pass" + ) + return + + model_version = context["model_version"] + meta = context["meta"] + label_matrix = context["label_matrix"] + thresholds = context["thresholds"] + signature = context["signature"] + logit_scale = context["logit_scale"] + logit_bias = context["logit_bias"] + + total_videos = 0 + start_time = time.time() + while True: + video_ids = db_get_videos_needing_scoring( + model_version, signature, SCORING_CHUNK_SIZE + ) + if not video_ids: + break + + for video_id in video_ids: + frame_matrix = db_get_frame_embeddings_for_video( + video_id, model_version + ) + if frame_matrix.size == 0: + # Can't happen given the query join, but stamping anyway + # keeps the while loop from spinning on such a row. + db_write_video_semantic_scores(video_id, [], signature) + continue + + logits = frame_matrix @ label_matrix.T * logit_scale + logit_bias + scores = 1.0 / (1.0 + np.exp(-logits)) # [frames, labels] + + hits = scores >= thresholds + frame_counts = hits.sum(axis=0) + best_scores = np.where(hits, scores, 0.0).max(axis=0) + + frame_total = frame_matrix.shape[0] + required = ( + VIDEO_TAG_MIN_FRAME_SUPPORT + if frame_total >= VIDEO_TAG_MIN_FRAME_SUPPORT + 1 + else 1 + ) + candidates = np.flatnonzero(frame_counts >= required) + if len(candidates) > VIDEO_TAG_TOP_K: + keep = np.argsort(best_scores[candidates])[::-1][:VIDEO_TAG_TOP_K] + candidates = candidates[keep] + + db_write_video_semantic_scores( + video_id, + [ + (meta[j][0], float(best_scores[j]), int(frame_counts[j])) + for j in candidates + ], + signature, + ) + total_videos += 1 + + if total_videos: + elapsed = time.time() - start_time + logger.info( + f"Video semantic scoring pass complete. Videos: {total_videos}, " + f"Labels: {len(meta)}, Elapsed: {elapsed:.2f}s" + ) + except Exception as e: + logger.error(f"Error in video semantic scoring pass: {e}") diff --git a/backend/app/utils/videos.py b/backend/app/utils/videos.py index 83930205c..68ce30193 100644 --- a/backend/app/utils/videos.py +++ b/backend/app/utils/videos.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import shutil import uuid import datetime import json @@ -11,7 +12,7 @@ import cv2 from PIL import Image -from app.config.settings import THUMBNAIL_IMAGES_PATH +from app.config.settings import THUMBNAIL_IMAGES_PATH, VIDEO_FRAMES_PATH from app.database.videos import ( VideoRecord, db_bulk_insert_videos, @@ -390,6 +391,352 @@ def video_util_remove_obsolete_videos(folder_id_list: List[int]) -> int: # Drop the rows first: an orphaned file beats a row pointing at nothing db_delete_videos_by_ids(obsolete_videos) video_util_remove_thumbnail_files(obsolete_thumbnails) + # The video_frames rows cascade with the video; their JPEGs don't. + for video_id in obsolete_videos: + video_util_remove_frame_directory(video_id) logger.info(f"Removed {len(obsolete_videos)} obsolete video(s) from database") return len(obsolete_videos) + + +# ============================================================================ +# KEYFRAME SAMPLING - AI tagging without per-frame inference +# ============================================================================ + + +def video_util_frame_directory(video_id: str) -> str: + """Where a video's sampled keyframe JPEGs live.""" + return os.path.join(VIDEO_FRAMES_PATH, video_id) + + +def video_util_remove_frame_directory(video_id: str) -> None: + frame_dir = video_util_frame_directory(video_id) + if os.path.isdir(frame_dir): + try: + shutil.rmtree(frame_dir) + except OSError as e: + logger.error(f"Error removing frame directory {frame_dir}: {e}") + + +def video_util_get_frame_interval() -> float: + """The user's keyframe interval, falling back to the configured default.""" + from app.config.settings import VIDEO_FRAME_INTERVAL_SECONDS + from app.database.metadata import db_get_metadata + + try: + metadata = db_get_metadata() or {} + interval = metadata.get("user_preferences", {}).get("Video_Frame_Interval") + if interval is not None: + return float(interval) + except Exception as e: + logger.warning(f"Could not read frame interval preference: {e}") + + return VIDEO_FRAME_INTERVAL_SECONDS + + +def video_util_sample_frame_timestamps( + duration: Optional[float], interval: float, max_frames: int +) -> List[float]: + """Timestamps (seconds) to sample a video at. + + Frames land at chunk midpoints -- the start of a video is often a black + leader frame, and a midpoint is the most representative moment of the + chunk it stands for. Videos long enough to exceed max_frames get their + interval stretched rather than their frame count grown, so inference cost + per video is bounded regardless of length. + """ + if not duration or duration <= 0 or interval <= 0: + return [0.0] + + effective_interval = max(interval, duration / max_frames) + if duration <= effective_interval: + return [duration / 2] + + timestamps = [] + position = effective_interval / 2 + while position < duration and len(timestamps) < max_frames: + timestamps.append(round(position, 3)) + position += effective_interval + + return timestamps or [duration / 2] + + +def video_util_extract_video_frames( + video_id: str, video_path: str, interval: float +) -> List[dict]: + """Sample a video into keyframe JPEGs on disk. + + Seeks to each timestamp rather than decoding sequentially -- that's what + keeps a two-hour video affordable. Returns frame records ready for + db_bulk_insert_video_frames; a frame that won't decode is skipped, not + fatal. + """ + from app.config.settings import ( + VIDEO_FRAME_MAX_DIMENSION, + VIDEO_MAX_FRAMES_PER_VIDEO, + ) + + frame_dir = video_util_frame_directory(video_id) + # Start from a clean sample: a re-tag must not mix frames from a previous + # interval into the new set. + video_util_remove_frame_directory(video_id) + + capture = cv2.VideoCapture(video_path) + try: + if not capture.isOpened(): + logger.warning(f"Could not open video for frame sampling: {video_path}") + return [] + + fps = capture.get(cv2.CAP_PROP_FPS) or 0 + frame_count = capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0 + duration = frame_count / fps if fps > 0 and frame_count > 0 else None + + timestamps = video_util_sample_frame_timestamps( + duration, interval, VIDEO_MAX_FRAMES_PER_VIDEO + ) + + os.makedirs(frame_dir, exist_ok=True) + + frame_records = [] + for index, timestamp in enumerate(timestamps): + capture.set(cv2.CAP_PROP_POS_MSEC, timestamp * 1000) + ret, frame = capture.read() + if not ret or frame is None: + continue + + frame_path = os.path.abspath( + os.path.join(frame_dir, f"frame_{index:04d}.jpg") + ) + try: + img = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) + img.thumbnail((VIDEO_FRAME_MAX_DIMENSION, VIDEO_FRAME_MAX_DIMENSION)) + img.save(frame_path, "JPEG", quality=85) + except Exception as e: + logger.error(f"Error saving frame {index} of {video_path}: {e}") + continue + + frame_records.append( + { + "id": str(uuid.uuid4()), + "video_id": video_id, + "frame_path": frame_path, + "timestamp_sec": timestamp, + "frame_index": index, + } + ) + + return frame_records + except Exception as e: + logger.error(f"Error sampling frames for {video_path}: {e}") + return [] + finally: + capture.release() + + +def video_util_aggregate_frame_classes( + frame_class_ids: List[List[int]], min_support: int +) -> List[Tuple[int, int]]: + """Roll per-frame detections up to video-level (class_id, frame_count). + + A class is only kept if it was seen in enough frames -- one detection in + one unlucky keyframe describes the frame, not the video. Videos too short + to have that many frames fall back to a support of 1. + """ + if not frame_class_ids: + return [] + + counts: Dict[int, int] = {} + for class_ids in frame_class_ids: + for class_id in set(class_ids or []): + counts[class_id] = counts.get(class_id, 0) + 1 + + required = min_support if len(frame_class_ids) >= min_support + 1 else 1 + return sorted( + ((class_id, count) for class_id, count in counts.items() if count >= required), + key=lambda pair: (-pair[1], pair[0]), + ) + + +def video_util_process_untagged_videos() -> bool: + """Sample and object-tag every untagged video in AI-tagging folders.""" + from app.config.settings import VIDEO_TAG_MIN_FRAME_SUPPORT + from app.database.video_frames import ( + db_bulk_insert_video_frames, + db_delete_frames_for_videos, + db_get_untagged_videos, + db_mark_videos_tagged, + db_write_video_classes, + ) + from app.models.ObjectClassifier import ObjectClassifier + + try: + untagged_videos = db_get_untagged_videos() + if not untagged_videos: + return True + + interval = video_util_get_frame_interval() + object_classifier = ObjectClassifier() + total_frames = 0 + + try: + for video in untagged_videos: + video_id = video["id"] + frames = video_util_extract_video_frames( + video_id, video["path"], interval + ) + + if not frames: + # Undecodable files are still marked tagged, otherwise + # every folder sync retries them forever. + logger.warning( + f"No frames sampled from {video['path']}; tagging as empty" + ) + db_mark_videos_tagged([video_id]) + continue + + db_delete_frames_for_videos([video_id]) + db_bulk_insert_video_frames(frames) + total_frames += len(frames) + + frame_class_ids = [ + object_classifier.get_classes(frame["frame_path"]) or [] + for frame in frames + ] + db_write_video_classes( + video_id, + video_util_aggregate_frame_classes( + frame_class_ids, VIDEO_TAG_MIN_FRAME_SUPPORT + ), + ) + db_mark_videos_tagged([video_id]) + finally: + object_classifier.close() + + logger.info( + f"Video tagging pass complete. Videos: {len(untagged_videos)}, " + f"Frames sampled: {total_frames}, Interval: {interval}s" + ) + return True + except Exception as e: + logger.error(f"Error processing untagged videos: {e}") + return False + + +def video_util_process_unembedded_frames() -> None: + """Embed sampled keyframes with SigLIP2 -- the video counterpart of + image_util_process_unembedded_images.""" + import time + + import numpy as np + + from app.config.settings import ( + SIGLIP2_ACTIVE_CHECKPOINT, + SIGLIP2_SCORING_METADATA, + SIGLIP2_EMBED_BATCH_SIZE, + ) + from app.database.video_frames import ( + db_get_unembedded_video_frames, + db_mark_video_frames_embedded, + db_upsert_video_frame_embeddings, + ) + from app.models.model_registry import get_siglip2_registry_keys, get_model_path + from app.models.SigLIP2Vision import SigLIP2Vision + from app.utils.SigLIP import siglip_util_preprocess_image + + try: + vision_key, _ = get_siglip2_registry_keys(SIGLIP2_ACTIVE_CHECKPOINT) + vision_model_path = get_model_path(vision_key) + if not os.path.exists(vision_model_path): + logger.info( + "SigLIP2 vision model not installed; skipping frame embedding pass" + ) + return + + unembedded_frames = db_get_unembedded_video_frames() + if not unembedded_frames: + return + + metadata = SIGLIP2_SCORING_METADATA[SIGLIP2_ACTIVE_CHECKPOINT] + resolution = metadata["input_resolution"] + model_version = metadata["model_version"] + + vision_model = SigLIP2Vision(vision_model_path) + try: + total_frames = len(unembedded_frames) + embedded_count = 0 + corrupt_count = 0 + start_time = time.time() + + for i in range(0, total_frames, SIGLIP2_EMBED_BATCH_SIZE): + batch = unembedded_frames[i : i + SIGLIP2_EMBED_BATCH_SIZE] + + good_arrays = [] + good_ids = [] + + for frame in batch: + preprocessed = siglip_util_preprocess_image( + frame["frame_path"], resolution + ) + if preprocessed is None: + corrupt_count += 1 + continue + + good_arrays.append(preprocessed) + good_ids.append(frame["id"]) + + if not good_arrays: + continue + + stacked = np.stack(good_arrays) # [N, 3, R, R] + embeddings = vision_model.get_embedding(stacked) # [N, D] + + db_upsert_video_frame_embeddings( + [ + (good_ids[idx], model_version, emb) + for idx, emb in enumerate(embeddings) + ] + ) + embedded_count += len(good_arrays) + + # Only frames that actually got an embedding row, so a frame + # that fails to decode is retried next pass. + db_mark_video_frames_embedded(good_ids) + + elapsed = time.time() - start_time + logger.info( + f"Video frame embedding pass complete. Total: {total_frames}, " + f"Embedded: {embedded_count}, Corrupt: {corrupt_count}, " + f"Elapsed: {elapsed:.2f}s" + ) + finally: + vision_model.close() + + except Exception as e: + logger.error(f"Error embedding video frames: {e}") + + +def video_util_purge_frame_cache() -> int: + """Delete the keyframe JPEGs and return the bytes reclaimed. + + Frame rows and their embeddings survive, so tags and semantic search keep + working -- only a re-tag would need the frames extracted again. + """ + from app.database.video_frames import db_clear_frame_paths + + reclaimed = 0 + if os.path.isdir(VIDEO_FRAMES_PATH): + for root, _, files in os.walk(VIDEO_FRAMES_PATH): + for file in files: + try: + reclaimed += os.path.getsize(os.path.join(root, file)) + except OSError: + continue + try: + shutil.rmtree(VIDEO_FRAMES_PATH) + except OSError as e: + logger.error(f"Error purging frame cache: {e}") + return 0 + + db_clear_frame_paths() + logger.info(f"Purged video frame cache, reclaimed {reclaimed} bytes") + return reclaimed diff --git a/backend/main.py b/backend/main.py index a36375f66..a6eb64327 100644 --- a/backend/main.py +++ b/backend/main.py @@ -24,10 +24,12 @@ from app.database.metadata import db_create_metadata_table from app.database.semantic_labels import db_create_semantic_labels_table from app.database.image_embeddings import db_create_image_embeddings_table +from app.database.video_frames import db_create_video_frames_tables from app.utils.semantic_labels import ( semantic_util_sync_vocabulary, semantic_util_build_label_embeddings, semantic_util_score_images, + semantic_util_score_videos, ) from app.routes.folders import router as folders_router @@ -67,6 +69,7 @@ async def lifespan(app: FastAPI): db_create_videos_table() db_create_semantic_labels_table() db_create_image_embeddings_table() + db_create_video_frames_tables() db_create_YOLO_classes_table() db_create_clusters_table() # Create clusters table first since faces references it db_create_faces_table() @@ -83,6 +86,7 @@ async def lifespan(app: FastAPI): # order: the scoring sweep needs the label embeddings. app.state.executor.submit(semantic_util_build_label_embeddings) app.state.executor.submit(semantic_util_score_images) + app.state.executor.submit(semantic_util_score_videos) # Start the SSE model download cleanup task cleanup_task = asyncio.create_task(_cleanup_stale_tasks()) diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 964977f87..72cce0cf0 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -12,6 +12,7 @@ from app.database.metadata import db_create_metadata_table from app.database.semantic_labels import db_create_semantic_labels_table from app.database.image_embeddings import db_create_image_embeddings_table +from app.database.video_frames import db_create_video_frames_tables @pytest.fixture(scope="session", autouse=True) @@ -34,6 +35,7 @@ def setup_before_all_tests(): db_create_videos_table() db_create_semantic_labels_table() db_create_image_embeddings_table() + db_create_video_frames_tables() db_create_metadata_table() print("All database tables created successfully") except Exception as e: From 3701d3f4de64f425a1857a29fe58794f6c1d4719 Mon Sep 17 00:00:00 2001 From: ROHAN PANDEY <95585299+rohan-pandeyy@users.noreply.github.com> Date: Fri, 24 Jul 2026 11:20:12 +0530 Subject: [PATCH 02/10] feat(backend): video tag & semantic search endpoints and frame-cache purge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Return aggregated tags from GET /videos/ and add: - GET /videos/search?tag= — videos carrying a tag - GET /videos/semantic-search?query= — SigLIP2 over keyframe embeddings, max-pooled per video with the best-matching frame's timestamp - POST /videos/purge-frame-cache — reclaim the on-disk frame JPEGs while keeping tags and embeddings intact Covered by test_video_frames.py (sampling, extraction, aggregation, purge, and the new routes). OpenAPI spec regenerated. --- backend/app/routes/videos.py | 257 +++++++++++-- backend/tests/test_video_frames.py | 448 +++++++++++++++++++++++ docs/backend/backend_python/openapi.json | 410 +++++++++++++++++++-- 3 files changed, 1056 insertions(+), 59 deletions(-) create mode 100644 backend/tests/test_video_frames.py diff --git a/backend/app/routes/videos.py b/backend/app/routes/videos.py index ff7230ed1..750b6dff9 100644 --- a/backend/app/routes/videos.py +++ b/backend/app/routes/videos.py @@ -1,9 +1,10 @@ -from fastapi import APIRouter, HTTPException, status +from fastapi import APIRouter, HTTPException, Query, status from typing import List, Optional from pydantic import BaseModel, ValidationError from app.database.videos import ( db_get_all_videos, + db_get_videos_by_ids, db_toggle_video_favourite_status, db_get_video_by_id, ) @@ -44,6 +45,30 @@ class GetAllVideosResponse(BaseModel): data: List[VideoData] +def _to_video_data(videos: List[dict]) -> List[VideoData]: + """Build per row: one record with unusable metadata shouldn't 500 the + whole listing and hide every other video.""" + video_data = [] + for video in videos: + try: + video_data.append( + VideoData( + id=video["id"], + path=video["path"], + folder_id=video["folder_id"], + thumbnailPath=video["thumbnailPath"], + metadata=video["metadata"], + isFavourite=video.get("isFavourite", False), + tags=video["tags"], + ) + ) + except ValidationError as e: + logger.warning( + f"Skipping video {video.get('id')} with invalid metadata: {e}" + ) + return video_data + + @router.get( "/", response_model=GetAllVideosResponse, @@ -52,28 +77,7 @@ class GetAllVideosResponse(BaseModel): def get_all_videos(): """Get all videos from the database.""" try: - videos = db_get_all_videos() - - # Build per row: one record with unusable metadata shouldn't 500 the - # whole listing and hide every other video. - video_data = [] - for video in videos: - try: - video_data.append( - VideoData( - id=video["id"], - path=video["path"], - folder_id=video["folder_id"], - thumbnailPath=video["thumbnailPath"], - metadata=video["metadata"], - isFavourite=video.get("isFavourite", False), - tags=video["tags"], - ) - ) - except ValidationError as e: - logger.warning( - f"Skipping video {video.get('id')} with invalid metadata: {e}" - ) + video_data = _to_video_data(db_get_all_videos()) return GetAllVideosResponse( success=True, @@ -92,6 +96,213 @@ def get_all_videos(): ) +@router.get( + "/search", + response_model=GetAllVideosResponse, + responses={code: {"model": ErrorResponse} for code in [400, 500]}, +) +def search_videos_by_tag(tag: str = Query(..., description="Tag name to search for")): + """Search videos by tag name.""" + try: + from app.database.video_frames import db_get_video_ids_by_tag + + video_data = _to_video_data(db_get_videos_by_ids(db_get_video_ids_by_tag(tag))) + + return GetAllVideosResponse( + success=True, + message=f"Successfully retrieved {len(video_data)} videos for tag '{tag}'", + data=video_data, + ) + + except Exception as e: + logger.error(f"Error searching videos: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=ErrorResponse( + success=False, + error="Internal server error", + message="Unable to search videos due to an internal error", + ).model_dump(), + ) + + +class ScoredVideoData(VideoData): + score: float + # When in the video the best match was found -- what a future "jump to + # this moment" deep link needs. + best_frame_timestamp: Optional[float] = None + + +class SemanticSearchData(BaseModel): + videos: List[ScoredVideoData] + total: int + + +class SemanticSearchResponse(BaseModel): + success: bool + message: str + data: SemanticSearchData + + +@router.get( + "/semantic-search", + response_model=SemanticSearchResponse, + responses={code: {"model": ErrorResponse} for code in [400, 404, 500]}, +) +def semantic_search_videos( + query: str = Query(..., min_length=1, description="Query text to search for") +): + """Semantic search videos by query text using SigLIP2 keyframe embeddings.""" + try: + import os + import numpy as np + + from app.config.settings import ( + SIGLIP2_ACTIVE_CHECKPOINT, + SIGLIP2_SCORING_METADATA, + SIGLIP2_MATCH_THRESHOLD, + SIGLIP2_QUERY_TEMPLATE, + ) + from app.database.video_frames import db_get_all_frame_embeddings + from app.models.model_registry import ( + get_siglip2_registry_keys, + get_siglip2_tokenizer_key, + get_model_path, + ) + from app.utils.SigLIP import ( + siglip_util_tokenize_query, + siglip_util_get_text_model, + ) + + _, text_key = get_siglip2_registry_keys(SIGLIP2_ACTIVE_CHECKPOINT) + tokenizer_key = get_siglip2_tokenizer_key(SIGLIP2_ACTIVE_CHECKPOINT) + text_model_path = get_model_path(text_key) + tokenizer_model_path = get_model_path(tokenizer_key) + + for path, missing in ( + (text_model_path, "SigLIP2 text model"), + (tokenizer_model_path, "SigLIP2 tokenizer"), + ): + if not os.path.exists(path): + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=ErrorResponse( + success=False, + error="Not Found", + message=f"semantic search unavailable: {missing} not installed", + ).model_dump(), + ) + + metadata = SIGLIP2_SCORING_METADATA[SIGLIP2_ACTIVE_CHECKPOINT] + video_ids, timestamps, frame_matrix = db_get_all_frame_embeddings( + metadata["model_version"] + ) + + if not video_ids: + return SemanticSearchResponse( + success=True, + message="No video frames have been embedded yet.", + data=SemanticSearchData(videos=[], total=0), + ) + + text_model = siglip_util_get_text_model(text_model_path, text_key) + input_ids, attention_mask = siglip_util_tokenize_query( + SIGLIP2_QUERY_TEMPLATE.format(query=query) + ) + query_vector = np.asarray( + text_model.get_embedding(input_ids, attention_mask), dtype=np.float32 + ).flatten() + + logits = ( + frame_matrix @ query_vector * np.exp(metadata["logit_scale"]) + + metadata["logit_bias"] + ) + scores = 1.0 / (1.0 + np.exp(-logits)) + + # Max-pool per video: a video matches as well as its best frame does. + best: dict = {} + for i, video_id in enumerate(video_ids): + score = float(scores[i]) + if score < SIGLIP2_MATCH_THRESHOLD: + continue + if video_id not in best or score > best[video_id][0]: + best[video_id] = (score, timestamps[i]) + + ranked = sorted(best.items(), key=lambda item: item[1][0], reverse=True) + # Keyed rather than zipped: db_get_videos_by_ids drops IDs it can't + # find, which would silently shift every score onto the wrong video. + by_id = { + video.id: video + for video in _to_video_data( + db_get_videos_by_ids([video_id for video_id, _ in ranked]) + ) + } + + scored = [ + ScoredVideoData( + **by_id[video_id].model_dump(), + score=score, + best_frame_timestamp=timestamp, + ) + for video_id, (score, timestamp) in ranked + if video_id in by_id + ] + + return SemanticSearchResponse( + success=True, + message=f"Found {len(scored)} matching videos", + data=SemanticSearchData(videos=scored, total=len(scored)), + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error in video semantic search: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=ErrorResponse( + success=False, + error="Internal server error", + message=f"Unable to search videos: {str(e)}", + ).model_dump(), + ) + + +class PurgeFrameCacheResponse(BaseModel): + success: bool + message: str + bytes_reclaimed: int + + +@router.post( + "/purge-frame-cache", + response_model=PurgeFrameCacheResponse, + responses={500: {"model": ErrorResponse}}, +) +def purge_frame_cache(): + """Delete the sampled keyframe JPEGs. Tags and semantic search survive: + only the on-disk frames are removed, not their embeddings.""" + try: + from app.utils.videos import video_util_purge_frame_cache + + reclaimed = video_util_purge_frame_cache() + return PurgeFrameCacheResponse( + success=True, + message=f"Reclaimed {reclaimed} bytes of video frame cache", + bytes_reclaimed=reclaimed, + ) + except Exception as e: + logger.error(f"Error purging video frame cache: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=ErrorResponse( + success=False, + error="Internal server error", + message=f"Unable to purge video frame cache: {str(e)}", + ).model_dump(), + ) + + class ToggleFavouriteRequest(BaseModel): video_id: str diff --git a/backend/tests/test_video_frames.py b/backend/tests/test_video_frames.py new file mode 100644 index 000000000..93a6131bc --- /dev/null +++ b/backend/tests/test_video_frames.py @@ -0,0 +1,448 @@ +import json +import os +import sqlite3 +import tempfile +import shutil + +import cv2 +import numpy as np +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from app.database.video_frames import ( + db_bulk_insert_video_frames, + db_clear_frame_paths, + db_create_video_frames_tables, + db_get_all_frame_embeddings, + db_get_frame_embeddings_for_video, + db_get_unembedded_video_frames, + db_get_untagged_videos, + db_get_video_ids_by_tag, + db_get_video_tags, + db_get_videos_needing_scoring, + db_mark_video_frames_embedded, + db_mark_videos_tagged, + db_upsert_video_frame_embeddings, + db_write_video_classes, + db_write_video_semantic_scores, +) +from app.database.videos import db_bulk_insert_videos, db_create_videos_table +from app.routes.videos import router as videos_router +from app.utils.videos import ( + video_util_aggregate_frame_classes, + video_util_extract_video_frames, + video_util_purge_frame_cache, + video_util_sample_frame_timestamps, +) + +# ############################## +# Pytest Fixtures +# ############################## + + +@pytest.fixture(scope="function") +def test_db(monkeypatch): + """Point every module that touches these tables at a fresh tempfile + database. video_frames borrows images._connect, so that module's + DATABASE_PATH has to be patched too.""" + db_fd, db_path = tempfile.mkstemp() + os.close(db_fd) + + monkeypatch.setattr("app.config.settings.DATABASE_PATH", db_path) + monkeypatch.setattr("app.database.images.DATABASE_PATH", db_path) + monkeypatch.setattr("app.database.videos.DATABASE_PATH", db_path) + monkeypatch.setattr("app.database.folders.DATABASE_PATH", db_path) + monkeypatch.setattr("app.database.yolo_mapping.DATABASE_PATH", db_path) + + from app.database.folders import db_create_folders_table + from app.database.yolo_mapping import db_create_YOLO_classes_table + + db_create_folders_table() + db_create_YOLO_classes_table() + db_create_videos_table() + db_create_video_frames_tables() + + yield db_path + + os.unlink(db_path) + + +@pytest.fixture +def frames_dir(monkeypatch): + """Redirect the frame cache at a temp dir so tests never touch the real + user data directory.""" + temp_dir = tempfile.mkdtemp() + monkeypatch.setattr("app.utils.videos.VIDEO_FRAMES_PATH", temp_dir) + yield temp_dir + shutil.rmtree(temp_dir, ignore_errors=True) + + +@pytest.fixture +def tagging_folder_id(test_db): + """A folder with AI tagging enabled, so its videos are pickable.""" + folder_id = "folder-ai-on" + conn = sqlite3.connect(test_db) + conn.execute( + "INSERT INTO folders (folder_id, folder_path, last_modified_time, AI_Tagging) " + "VALUES (?, ?, 0, 1)", + (folder_id, os.path.abspath("tagging-folder")), + ) + conn.commit() + conn.close() + return folder_id + + +@pytest.fixture +def video_id(tagging_folder_id): + video = { + "id": "vid-1", + "path": os.path.abspath("tagging-folder/clip.mp4"), + "folder_id": tagging_folder_id, + "thumbnailPath": "/thumbs/thumbnail_vid-1.jpg", + "metadata": json.dumps( + { + "name": "clip.mp4", + "date_created": "2026-01-01T00:00:00", + "width": 1920, + "height": 1080, + "duration": 30.0, + "fps": 30.0, + "file_location": "clip.mp4", + "file_size": 1024, + "item_type": "video/mp4", + } + ), + "isTagged": False, + "captured_at": "2026-01-01T00:00:00", + } + db_bulk_insert_videos([video]) + return video["id"] + + +@pytest.fixture +def temp_media_dir(): + temp_dir = tempfile.mkdtemp() + yield temp_dir + shutil.rmtree(temp_dir, ignore_errors=True) + + +@pytest.fixture +def real_video_file(temp_media_dir): + """A 3-second video: long enough that a 1s interval yields several frames.""" + video_path = os.path.join(temp_media_dir, "sample.avi") + writer = cv2.VideoWriter( + video_path, cv2.VideoWriter_fourcc(*"MJPG"), 10.0, (64, 64) + ) + if not writer.isOpened(): + pytest.skip("cv2.VideoWriter unavailable in this environment") + for i in range(30): + writer.write(np.full((64, 64, 3), (i * 8) % 256, dtype=np.uint8)) + writer.release() + return video_path + + +@pytest.fixture +def client(test_db): + app = FastAPI() + app.include_router(videos_router, prefix="/videos") + return TestClient(app) + + +def insert_frames(video_id, count, frames_dir=None): + """Insert `count` frame rows, one second apart.""" + records = [ + { + "id": f"{video_id}-frame-{i}", + "video_id": video_id, + "frame_path": os.path.join(frames_dir or "/frames", f"f{i}.jpg"), + "timestamp_sec": float(i), + "frame_index": i, + } + for i in range(count) + ] + db_bulk_insert_video_frames(records) + return records + + +# ############################## +# Sampling strategy +# ############################## + + +class TestSampleFrameTimestamps: + def test_samples_at_chunk_midpoints(self): + # 20s at 5s intervals: 2.5, 7.5, 12.5, 17.5 -- never the leading frame, + # which is often black. + assert video_util_sample_frame_timestamps(20.0, 5.0, 200) == [ + 2.5, + 7.5, + 12.5, + 17.5, + ] + + def test_ten_minute_video_at_five_seconds(self): + assert len(video_util_sample_frame_timestamps(600.0, 5.0, 200)) == 120 + + def test_long_video_stretches_interval_instead_of_frame_count(self): + timestamps = video_util_sample_frame_timestamps(3 * 3600.0, 5.0, 200) + assert len(timestamps) == 200 + # 3 hours over 200 frames = one frame per 54 seconds. + assert timestamps[1] - timestamps[0] == pytest.approx(54.0) + + def test_video_shorter_than_the_interval_yields_one_midpoint_frame(self): + assert video_util_sample_frame_timestamps(3.0, 5.0, 200) == [1.5] + + @pytest.mark.parametrize("duration", [None, 0, -1]) + def test_unknown_duration_falls_back_to_a_single_frame(self, duration): + assert video_util_sample_frame_timestamps(duration, 5.0, 200) == [0.0] + + def test_respects_the_cap_exactly(self): + assert len(video_util_sample_frame_timestamps(100.0, 1.0, 10)) == 10 + + +# ############################## +# Aggregating frames into video tags +# ############################## + + +class TestAggregateFrameClasses: + def test_drops_classes_seen_in_too_few_frames(self): + frames = [[0]] * 4 + [[15]] # person in 4 frames, cat in 1 + assert video_util_aggregate_frame_classes(frames, 2) == [(0, 4)] + + def test_counts_a_class_once_per_frame(self): + # Three people in one frame is still one frame of support. + frames = [[0, 0, 0], [0, 0]] + assert video_util_aggregate_frame_classes(frames, 2) == [(0, 2)] + + def test_short_videos_relax_the_support_requirement(self): + # Two frames can never reach a support of 2 for a class in one of them, + # so the requirement drops to 1 rather than tagging nothing. + assert video_util_aggregate_frame_classes([[0], [15]], 2) == [(0, 1), (15, 1)] + + def test_orders_by_frame_count_descending(self): + frames = [[0], [0, 15], [0, 15], [15]] + assert video_util_aggregate_frame_classes(frames, 2) == [(0, 3), (15, 3)] + + def test_no_frames_means_no_tags(self): + assert video_util_aggregate_frame_classes([], 2) == [] + + +# ############################## +# Frame extraction +# ############################## + + +class TestExtractVideoFrames: + def test_writes_a_jpeg_per_sampled_timestamp(self, real_video_file, frames_dir): + records = video_util_extract_video_frames("vid-1", real_video_file, 1.0) + + assert len(records) == 3 # 3 seconds of footage at 1s intervals + for record in records: + assert os.path.exists(record["frame_path"]) + assert record["video_id"] == "vid-1" + assert [r["frame_index"] for r in records] == [0, 1, 2] + + def test_frames_are_downscaled(self, real_video_file, frames_dir, monkeypatch): + monkeypatch.setattr("app.config.settings.VIDEO_FRAME_MAX_DIMENSION", 32) + records = video_util_extract_video_frames("vid-1", real_video_file, 1.0) + + frame = cv2.imread(records[0]["frame_path"]) + assert max(frame.shape[:2]) <= 32 + + def test_resampling_replaces_the_previous_frames(self, real_video_file, frames_dir): + dense = video_util_extract_video_frames("vid-1", real_video_file, 1.0) + sparse = video_util_extract_video_frames("vid-1", real_video_file, 10.0) + + assert len(sparse) < len(dense) + # The old dense sample is gone rather than left behind as orphans. + frame_dir = os.path.join(frames_dir, "vid-1") + assert len(os.listdir(frame_dir)) == len(sparse) + + def test_undecodable_file_yields_no_frames(self, temp_media_dir, frames_dir): + broken = os.path.join(temp_media_dir, "broken.mp4") + with open(broken, "wb") as f: + f.write(b"not a video") + + assert video_util_extract_video_frames("vid-1", broken, 5.0) == [] + + +# ############################## +# Database round-trips +# ############################## + + +class TestVideoFrameDatabase: + def test_untagged_videos_only_from_ai_tagging_folders(self, video_id): + assert [v["id"] for v in db_get_untagged_videos()] == [video_id] + + db_mark_videos_tagged([video_id]) + assert db_get_untagged_videos() == [] + + def test_frames_cascade_when_their_video_is_deleted(self, video_id, test_db): + insert_frames(video_id, 3) + + from app.database.videos import db_delete_videos_by_ids + + db_delete_videos_by_ids([video_id]) + + conn = sqlite3.connect(test_db) + count = conn.execute("SELECT COUNT(*) FROM video_frames").fetchone()[0] + conn.close() + assert count == 0 + + def test_embedded_frames_drop_out_of_the_pending_list(self, video_id): + frames = insert_frames(video_id, 3) + assert len(db_get_unembedded_video_frames()) == 3 + + db_mark_video_frames_embedded([frames[0]["id"]]) + assert len(db_get_unembedded_video_frames()) == 2 + + def test_purged_frames_are_not_re_embedded(self, video_id): + insert_frames(video_id, 3) + db_clear_frame_paths() + + # Nothing on disk to read, so they must not come back as pending work. + assert db_get_unembedded_video_frames() == [] + + def test_semantic_rescoring_leaves_yolo_tags_alone(self, video_id, test_db): + conn = sqlite3.connect(test_db) + conn.execute("INSERT INTO mappings (class_id, name) VALUES (1000, 'sunset')") + conn.commit() + conn.close() + + db_write_video_classes(video_id, [(0, 4)]) + db_write_video_semantic_scores(video_id, [(1000, 0.9, 3)], "sig-1") + assert sorted(db_get_video_tags([video_id])[video_id]) == [ + "person", + "sunset", + ] + + # A vocabulary change rewrites the semantic rows only. + db_write_video_semantic_scores(video_id, [], "sig-2") + assert db_get_video_tags([video_id])[video_id] == ["person"] + + def test_tag_lookup_is_case_insensitive(self, video_id): + db_write_video_classes(video_id, [(0, 4)]) + + assert db_get_video_ids_by_tag("PERSON") == [video_id] + assert db_get_video_ids_by_tag("giraffe") == [] + + def test_embeddings_round_trip_per_video_and_globally(self, video_id): + frames = insert_frames(video_id, 2) + db_upsert_video_frame_embeddings( + [ + (frames[0]["id"], "m1", np.array([1.0, 0.0], dtype=np.float32)), + (frames[1]["id"], "m1", np.array([0.0, 1.0], dtype=np.float32)), + ] + ) + + matrix = db_get_frame_embeddings_for_video(video_id, "m1") + assert matrix.shape == (2, 2) + + video_ids, timestamps, all_matrix = db_get_all_frame_embeddings("m1") + assert video_ids == [video_id, video_id] + assert timestamps == [0.0, 1.0] + assert all_matrix.shape == (2, 2) + + def test_scoring_signature_gates_rework(self, video_id): + frames = insert_frames(video_id, 1) + db_upsert_video_frame_embeddings( + [(frames[0]["id"], "m1", np.array([1.0, 0.0], dtype=np.float32))] + ) + + assert db_get_videos_needing_scoring("m1", "sig-1", 10) == [video_id] + + db_write_video_semantic_scores(video_id, [], "sig-1") + assert db_get_videos_needing_scoring("m1", "sig-1", 10) == [] + # A vocabulary change invalidates the stamp and the work comes back. + assert db_get_videos_needing_scoring("m1", "sig-2", 10) == [video_id] + + +# ############################## +# Purging the frame cache +# ############################## + + +class TestPurgeFrameCache: + def test_removes_jpegs_but_keeps_tags_and_embeddings( + self, video_id, real_video_file, frames_dir + ): + frames = video_util_extract_video_frames("vid-1", real_video_file, 1.0) + db_bulk_insert_video_frames(frames) + db_upsert_video_frame_embeddings( + [(frames[0]["id"], "m1", np.array([1.0, 0.0], dtype=np.float32))] + ) + db_write_video_classes(video_id, [(0, 3)]) + + reclaimed = video_util_purge_frame_cache() + + assert reclaimed > 0 + assert not os.path.exists(frames_dir) + # The index survives the cache: tags and semantic search keep working. + assert db_get_video_tags([video_id])[video_id] == ["person"] + assert db_get_frame_embeddings_for_video(video_id, "m1").shape == (1, 2) + + def test_purging_an_empty_cache_is_harmless(self, test_db, frames_dir): + shutil.rmtree(frames_dir, ignore_errors=True) + assert video_util_purge_frame_cache() == 0 + + +# ############################## +# Routes +# ############################## + + +class TestVideoTagRoutes: + def test_get_all_videos_returns_tags(self, client, video_id): + db_write_video_classes(video_id, [(0, 4)]) + + response = client.get("/videos/") + + assert response.status_code == 200 + assert response.json()["data"][0]["tags"] == ["person"] + + def test_untagged_video_reports_null_tags(self, client, video_id): + response = client.get("/videos/") + + assert response.json()["data"][0]["tags"] is None + + def test_search_by_tag(self, client, video_id): + db_write_video_classes(video_id, [(0, 4)]) + + response = client.get("/videos/search", params={"tag": "person"}) + + assert response.status_code == 200 + assert [v["id"] for v in response.json()["data"]] == [video_id] + + def test_search_by_tag_with_no_matches(self, client, video_id): + response = client.get("/videos/search", params={"tag": "person"}) + + assert response.status_code == 200 + assert response.json()["data"] == [] + + def test_semantic_search_without_models_returns_404( + self, client, video_id, monkeypatch + ): + monkeypatch.setattr( + "app.models.model_registry.get_model_path", + lambda key: "/nonexistent/model.onnx", + ) + + response = client.get("/videos/semantic-search", params={"query": "a beach"}) + + assert response.status_code == 404 + assert "not installed" in response.json()["detail"]["message"] + + def test_purge_route_reports_bytes_reclaimed( + self, client, video_id, real_video_file, frames_dir + ): + db_bulk_insert_video_frames( + video_util_extract_video_frames("vid-1", real_video_file, 1.0) + ) + + response = client.post("/videos/purge-frame-cache") + + assert response.status_code == 200 + assert response.json()["bytes_reclaimed"] > 0 diff --git a/docs/backend/backend_python/openapi.json b/docs/backend/backend_python/openapi.json index 959547a00..3122def49 100644 --- a/docs/backend/backend_python/openapi.json +++ b/docs/backend/backend_python/openapi.json @@ -990,7 +990,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SemanticSearchResponse" + "$ref": "#/components/schemas/app__routes__images__SemanticSearchResponse" } } } @@ -1110,6 +1110,179 @@ } } }, + "/videos/search": { + "get": { + "tags": [ + "Videos" + ], + "summary": "Search Videos By Tag", + "description": "Search videos by tag name.", + "operationId": "search_videos_by_tag_videos_search_get", + "parameters": [ + { + "name": "tag", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "Tag name to search for", + "title": "Tag" + }, + "description": "Tag name to search for" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GetAllVideosResponse" + } + } + } + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/app__schemas__videos__ErrorResponse" + } + } + }, + "description": "Bad Request" + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/app__schemas__videos__ErrorResponse" + } + } + }, + "description": "Internal Server Error" + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/videos/semantic-search": { + "get": { + "tags": [ + "Videos" + ], + "summary": "Semantic Search Videos", + "description": "Semantic search videos by query text using SigLIP2 keyframe embeddings.", + "operationId": "semantic_search_videos_videos_semantic_search_get", + "parameters": [ + { + "name": "query", + "in": "query", + "required": true, + "schema": { + "type": "string", + "minLength": 1, + "description": "Query text to search for", + "title": "Query" + }, + "description": "Query text to search for" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/app__routes__videos__SemanticSearchResponse" + } + } + } + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/app__schemas__videos__ErrorResponse" + } + } + }, + "description": "Bad Request" + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/app__schemas__videos__ErrorResponse" + } + } + }, + "description": "Not Found" + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/app__schemas__videos__ErrorResponse" + } + } + }, + "description": "Internal Server Error" + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/videos/purge-frame-cache": { + "post": { + "tags": [ + "Videos" + ], + "summary": "Purge Frame Cache", + "description": "Delete the sampled keyframe JPEGs. Tags and semantic search survive:\nonly the on-disk frames are removed, not their embeddings.", + "operationId": "purge_frame_cache_videos_purge_frame_cache_post", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PurgeFrameCacheResponse" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/app__schemas__videos__ErrorResponse" + } + } + } + } + } + } + }, "/videos/toggle-favourite": { "post": { "tags": [ @@ -2458,6 +2631,11 @@ "type": "integer", "title": "Image Count", "default": 0 + }, + "video_count": { + "type": "integer", + "title": "Video Count", + "default": 0 } }, "type": "object", @@ -3297,6 +3475,29 @@ ], "title": "MultiPersonSearchResponse" }, + "PurgeFrameCacheResponse": { + "properties": { + "success": { + "type": "boolean", + "title": "Success" + }, + "message": { + "type": "string", + "title": "Message" + }, + "bytes_reclaimed": { + "type": "integer", + "title": "Bytes Reclaimed" + } + }, + "type": "object", + "required": [ + "success", + "message", + "bytes_reclaimed" + ], + "title": "PurgeFrameCacheResponse" + }, "RenameClusterData": { "properties": { "cluster_id": { @@ -3373,26 +3574,79 @@ ], "title": "RenameClusterResponse" }, - "SemanticSearchData": { + "ScoredVideoData": { "properties": { - "images": { - "items": { - "$ref": "#/components/schemas/SemanticSearchImage" - }, - "type": "array", - "title": "Images" + "id": { + "type": "string", + "title": "Id" }, - "total": { - "type": "integer", - "title": "Total" + "path": { + "type": "string", + "title": "Path" + }, + "folder_id": { + "type": "string", + "title": "Folder Id" + }, + "thumbnailPath": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Thumbnailpath" + }, + "metadata": { + "$ref": "#/components/schemas/VideoMetadataModel" + }, + "isFavourite": { + "type": "boolean", + "title": "Isfavourite" + }, + "tags": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Tags" + }, + "score": { + "type": "number", + "title": "Score" + }, + "best_frame_timestamp": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Best Frame Timestamp" } }, "type": "object", "required": [ - "images", - "total" + "id", + "path", + "folder_id", + "thumbnailPath", + "metadata", + "isFavourite", + "score" ], - "title": "SemanticSearchData" + "title": "ScoredVideoData" }, "SemanticSearchImage": { "properties": { @@ -3455,28 +3709,6 @@ ], "title": "SemanticSearchImage" }, - "SemanticSearchResponse": { - "properties": { - "success": { - "type": "boolean", - "title": "Success" - }, - "message": { - "type": "string", - "title": "Message" - }, - "data": { - "$ref": "#/components/schemas/SemanticSearchData" - } - }, - "type": "object", - "required": [ - "success", - "message", - "data" - ], - "title": "SemanticSearchResponse" - }, "SetupRequest": { "properties": { "tier": { @@ -3819,6 +4051,19 @@ } ], "title": "Gpu Acceleration" + }, + "Video_Frame_Interval": { + "anyOf": [ + { + "type": "number", + "maximum": 60.0, + "minimum": 1.0 + }, + { + "type": "null" + } + ], + "title": "Video Frame Interval" } }, "type": "object", @@ -3864,6 +4109,13 @@ "type": "boolean", "title": "Gpu Acceleration", "default": true + }, + "Video_Frame_Interval": { + "type": "number", + "maximum": 60.0, + "minimum": 1.0, + "title": "Video Frame Interval", + "default": 5.0 } }, "type": "object", @@ -4033,6 +4285,49 @@ ], "title": "VideoMetadataModel" }, + "app__routes__images__SemanticSearchData": { + "properties": { + "images": { + "items": { + "$ref": "#/components/schemas/SemanticSearchImage" + }, + "type": "array", + "title": "Images" + }, + "total": { + "type": "integer", + "title": "Total" + } + }, + "type": "object", + "required": [ + "images", + "total" + ], + "title": "SemanticSearchData" + }, + "app__routes__images__SemanticSearchResponse": { + "properties": { + "success": { + "type": "boolean", + "title": "Success" + }, + "message": { + "type": "string", + "title": "Message" + }, + "data": { + "$ref": "#/components/schemas/app__routes__images__SemanticSearchData" + } + }, + "type": "object", + "required": [ + "success", + "message", + "data" + ], + "title": "SemanticSearchResponse" + }, "app__routes__images__ToggleFavouriteRequest": { "properties": { "image_id": { @@ -4046,6 +4341,49 @@ ], "title": "ToggleFavouriteRequest" }, + "app__routes__videos__SemanticSearchData": { + "properties": { + "videos": { + "items": { + "$ref": "#/components/schemas/ScoredVideoData" + }, + "type": "array", + "title": "Videos" + }, + "total": { + "type": "integer", + "title": "Total" + } + }, + "type": "object", + "required": [ + "videos", + "total" + ], + "title": "SemanticSearchData" + }, + "app__routes__videos__SemanticSearchResponse": { + "properties": { + "success": { + "type": "boolean", + "title": "Success" + }, + "message": { + "type": "string", + "title": "Message" + }, + "data": { + "$ref": "#/components/schemas/app__routes__videos__SemanticSearchData" + } + }, + "type": "object", + "required": [ + "success", + "message", + "data" + ], + "title": "SemanticSearchResponse" + }, "app__routes__videos__ToggleFavouriteRequest": { "properties": { "video_id": { From 11b4ac09bdb48ea383eb3896a37be6e34de25708 Mon Sep 17 00:00:00 2001 From: ROHAN PANDEY <95585299+rohan-pandeyy@users.noreply.github.com> Date: Fri, 24 Jul 2026 11:20:27 +0530 Subject: [PATCH 03/10] feat(backend): configurable video frame-sampling interval preference Add Video_Frame_Interval (seconds, 1-60) to user preferences, stored in the metadata blob alongside the existing settings. The tagging pass reads it to decide how often to sample keyframes, falling back to the configured default. --- backend/app/routes/user_preferences.py | 11 ++++++++++- backend/app/schemas/user_preferences.py | 6 +++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/backend/app/routes/user_preferences.py b/backend/app/routes/user_preferences.py index 678e8cfc0..229d121ce 100644 --- a/backend/app/routes/user_preferences.py +++ b/backend/app/routes/user_preferences.py @@ -30,6 +30,7 @@ def get_user_preferences(): user_preferences = UserPreferencesData( YOLO_model_size=user_prefs_data.get("YOLO_model_size", "small"), GPU_Acceleration=user_prefs_data.get("GPU_Acceleration", True), + Video_Frame_Interval=user_prefs_data.get("Video_Frame_Interval", 5.0), ) return GetUserPreferencesResponse( @@ -58,7 +59,11 @@ def update_user_preferences(request: UpdateUserPreferencesRequest): """Update user preferences in metadata.""" try: # Step 1: Validate that at least one field is provided - if request.YOLO_model_size is None and request.GPU_Acceleration is None: + if ( + request.YOLO_model_size is None + and request.GPU_Acceleration is None + and request.Video_Frame_Interval is None + ): raise ValueError("At least one preference field must be provided") # Step 2: Get current metadata @@ -74,6 +79,9 @@ def update_user_preferences(request: UpdateUserPreferencesRequest): if request.GPU_Acceleration is not None: current_user_prefs["GPU_Acceleration"] = request.GPU_Acceleration + if request.Video_Frame_Interval is not None: + current_user_prefs["Video_Frame_Interval"] = request.Video_Frame_Interval + # Step 5: Update metadata with new user preferences metadata["user_preferences"] = current_user_prefs @@ -94,6 +102,7 @@ def update_user_preferences(request: UpdateUserPreferencesRequest): user_preferences = UserPreferencesData( YOLO_model_size=current_user_prefs.get("YOLO_model_size", "small"), GPU_Acceleration=current_user_prefs.get("GPU_Acceleration", True), + Video_Frame_Interval=current_user_prefs.get("Video_Frame_Interval", 5.0), ) return UpdateUserPreferencesResponse( diff --git a/backend/app/schemas/user_preferences.py b/backend/app/schemas/user_preferences.py index d24191d78..094698e85 100644 --- a/backend/app/schemas/user_preferences.py +++ b/backend/app/schemas/user_preferences.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel +from pydantic import BaseModel, Field from typing import Optional, Literal @@ -7,6 +7,9 @@ class UserPreferencesData(BaseModel): YOLO_model_size: Literal["nano", "small", "medium"] = "small" GPU_Acceleration: bool = True + # Seconds between sampled video keyframes. Lower means finer tag coverage + # and proportionally more inference per video. + Video_Frame_Interval: float = Field(default=5.0, ge=1.0, le=60.0) class GetUserPreferencesResponse(BaseModel): @@ -22,6 +25,7 @@ class UpdateUserPreferencesRequest(BaseModel): YOLO_model_size: Optional[Literal["nano", "small", "medium"]] = None GPU_Acceleration: Optional[bool] = None + Video_Frame_Interval: Optional[float] = Field(default=None, ge=1.0, le=60.0) class UpdateUserPreferencesResponse(BaseModel): From 4a322ab1224b3fc8433e34599ec668cddb746e89 Mon Sep 17 00:00:00 2001 From: ROHAN PANDEY <95585299+rohan-pandeyy@users.noreply.github.com> Date: Fri, 24 Jul 2026 11:20:56 +0530 Subject: [PATCH 04/10] feat(frontend): video tags, search, and settings controls - Tag chips on the video card and a Tags row in the video info panel. - Wire the search page to run video tag and semantic search alongside images, rendering matching videos in their own section. - Add a keyframe-interval control and a "clear video frame cache" action to the user preferences card. --- .../src/api/api-functions/user_preferences.ts | 3 + frontend/src/api/api-functions/videos.ts | 48 ++++++ frontend/src/api/apiEndpoints.ts | 4 + frontend/src/components/Media/ImageTags.tsx | 9 +- frontend/src/components/Media/VideoCard.tsx | 12 ++ .../Media/__tests__/VideoCard.test.tsx | 18 +++ .../components/VideoPlayer/VideoInfoPanel.tsx | 24 ++- frontend/src/hooks/useUserPreferences.tsx | 13 ++ .../src/pages/SearchResults/SearchResults.tsx | 120 ++++++++++++-- .../components/UserPreferencesCard.tsx | 149 +++++++++++++++++- .../pages/__tests__/SearchResults.test.tsx | 48 +++++- frontend/src/types/Media.ts | 6 + 12 files changed, 428 insertions(+), 26 deletions(-) diff --git a/frontend/src/api/api-functions/user_preferences.ts b/frontend/src/api/api-functions/user_preferences.ts index bc0a02af7..391a01749 100644 --- a/frontend/src/api/api-functions/user_preferences.ts +++ b/frontend/src/api/api-functions/user_preferences.ts @@ -6,6 +6,8 @@ import { APIResponse } from '@/types/API'; export interface UserPreferencesData { YOLO_model_size: 'nano' | 'small' | 'medium'; GPU_Acceleration: boolean; + /** Seconds between sampled video keyframes when tagging videos. */ + Video_Frame_Interval: number; } export interface GetUserPreferencesResponse extends APIResponse { @@ -15,6 +17,7 @@ export interface GetUserPreferencesResponse extends APIResponse { export interface UpdateUserPreferencesRequest { YOLO_model_size?: 'nano' | 'small' | 'medium'; GPU_Acceleration?: boolean; + Video_Frame_Interval?: number; } export interface UpdateUserPreferencesResponse extends APIResponse { diff --git a/frontend/src/api/api-functions/videos.ts b/frontend/src/api/api-functions/videos.ts index 85fe3293d..a48e0ae5d 100644 --- a/frontend/src/api/api-functions/videos.ts +++ b/frontend/src/api/api-functions/videos.ts @@ -1,6 +1,7 @@ import { videosEndpoints } from '../apiEndpoints'; import { apiClient } from '../axiosConfig'; import { APIResponse } from '@/types/API'; +import { ScoredVideo } from '@/types/Media'; export const fetchAllVideos = async (): Promise => { const response = await apiClient.get( @@ -18,3 +19,50 @@ export const toggleVideoFav = async ( ); return response.data; }; + +export interface SearchVideosByTagRequest { + tag: string; +} + +export const searchVideosByTag = async ( + request: SearchVideosByTagRequest, +): Promise => { + const response = await apiClient.get( + videosEndpoints.searchByTag(request.tag), + ); + return response.data; +}; + +export interface SemanticSearchVideosRequest { + query: string; +} + +export interface SemanticSearchVideosAPIResponse extends APIResponse { + data?: { + videos: ScoredVideo[]; + total: number; + }; +} + +export const semanticSearchVideos = async ( + request: SemanticSearchVideosRequest, +): Promise => { + const response = await apiClient.get( + videosEndpoints.semanticSearch(request.query), + ); + return response.data; +}; + +export interface PurgeFrameCacheResponse { + success: boolean; + message: string; + bytes_reclaimed: number; +} + +export const purgeVideoFrameCache = + async (): Promise => { + const response = await apiClient.post( + videosEndpoints.purgeFrameCache, + ); + return response.data; + }; diff --git a/frontend/src/api/apiEndpoints.ts b/frontend/src/api/apiEndpoints.ts index 10a380902..b1b5a21a3 100644 --- a/frontend/src/api/apiEndpoints.ts +++ b/frontend/src/api/apiEndpoints.ts @@ -9,6 +9,10 @@ export const imagesEndpoints = { export const videosEndpoints = { getAllVideos: '/videos/', setFavourite: '/videos/toggle-favourite', + searchByTag: (tag: string) => `/videos/search?tag=${encodeURIComponent(tag)}`, + semanticSearch: (query: string) => + `/videos/semantic-search?query=${encodeURIComponent(query)}`, + purgeFrameCache: '/videos/purge-frame-cache', }; export const faceClustersEndpoints = { diff --git a/frontend/src/components/Media/ImageTags.tsx b/frontend/src/components/Media/ImageTags.tsx index bd0b0da79..7598a64b5 100644 --- a/frontend/src/components/Media/ImageTags.tsx +++ b/frontend/src/components/Media/ImageTags.tsx @@ -1,4 +1,5 @@ import { Badge } from '@/components/ui/badge'; +import { cn } from '@/lib/utils'; import { Tag } from 'lucide-react'; import { useState, useEffect, useRef } from 'react'; @@ -6,12 +7,15 @@ interface ImageTagsProps { tags: any[]; showTags?: boolean; isImageHovered: boolean; + /** Extra classes for the anchor, e.g. a z-index above overlay controls. */ + className?: string; } export function ImageTags({ tags, showTags = true, isImageHovered, + className, }: ImageTagsProps) { const [isTagsHovered, setIsTagsHovered] = useState(false); const [showTagsFromImageHover, setShowTagsFromImageHover] = useState(false); @@ -50,7 +54,10 @@ export function ImageTags({ return (
showTags && setIsTagsHovered(true)} onMouseLeave={() => showTags && setIsTagsHovered(false)} > diff --git a/frontend/src/components/Media/VideoCard.tsx b/frontend/src/components/Media/VideoCard.tsx index 222a3e55a..bab83879c 100644 --- a/frontend/src/components/Media/VideoCard.tsx +++ b/frontend/src/components/Media/VideoCard.tsx @@ -7,6 +7,7 @@ import { Video } from '@/types/Media'; import { convertFileSrc } from '@tauri-apps/api/core'; import { useToggleVideoFav } from '@/hooks/useToggleVideoFav'; import { formatDurationLabel } from '@/utils/durationUtils'; +import { ImageTags } from '@/components/Media/ImageTags'; interface VideoCardProps { video: Video; @@ -20,6 +21,7 @@ export function VideoCard({ video, className, onClick }: VideoCardProps) { // A rescan can replace the poster on disk, so fall back rather than showing // a broken image if the file is gone by the time the card renders. const [thumbnailFailed, setThumbnailFailed] = useState(false); + const [isHovered, setIsHovered] = useState(false); useEffect(() => setThumbnailFailed(false), [video.thumbnailPath]); @@ -35,6 +37,8 @@ export function VideoCard({ video, className, onClick }: VideoCardProps) { 'group bg-card overflow-hidden rounded-lg border transition-all hover:shadow-md', className, )} + onMouseEnter={() => setIsHovered(true)} + onMouseLeave={() => setIsHovered(false)} >
@@ -109,6 +113,14 @@ export function VideoCard({ video, className, onClick }: VideoCardProps) {
)} + + {/* Tags sampled from the video's keyframes. Stacked above the + playback overlay so the badge stays hoverable. */} +
); diff --git a/frontend/src/components/Media/__tests__/VideoCard.test.tsx b/frontend/src/components/Media/__tests__/VideoCard.test.tsx index 7e8e07350..324f6a21f 100644 --- a/frontend/src/components/Media/__tests__/VideoCard.test.tsx +++ b/frontend/src/components/Media/__tests__/VideoCard.test.tsx @@ -132,4 +132,22 @@ describe('VideoCard', () => { fireEvent.click(favourite); expect(mockToggleFavourite).not.toHaveBeenCalled(); }); + + test('shows a tag count badge for tagged videos, and the tags on hover', async () => { + render(); + + // Collapsed: just the count. + expect(screen.getByText('2')).toBeInTheDocument(); + expect(screen.queryByText('beach')).not.toBeInTheDocument(); + + fireEvent.mouseEnter(screen.getByText('2')); + expect(await screen.findByText('beach')).toBeInTheDocument(); + expect(screen.getByText('sunset')).toBeInTheDocument(); + }); + + test('renders no tag badge when the video has no tags', () => { + render(); + + expect(screen.queryByText('0')).not.toBeInTheDocument(); + }); }); diff --git a/frontend/src/components/VideoPlayer/VideoInfoPanel.tsx b/frontend/src/components/VideoPlayer/VideoInfoPanel.tsx index 8788cd8da..9f248ab74 100644 --- a/frontend/src/components/VideoPlayer/VideoInfoPanel.tsx +++ b/frontend/src/components/VideoPlayer/VideoInfoPanel.tsx @@ -1,6 +1,6 @@ import React from 'react'; import { motion, AnimatePresence } from 'framer-motion'; -import { X, Film, Calendar, Clock, Monitor, Info } from 'lucide-react'; +import { X, Film, Calendar, Clock, Monitor, Info, Tag } from 'lucide-react'; import { Video } from '@/types/Media'; import { formatDurationLabel } from '@/utils/durationUtils'; @@ -41,6 +41,7 @@ export const VideoInfoPanel: React.FC = ({ const height = video?.metadata?.height ?? 0; const resolution = width > 0 && height > 0 ? `${width} × ${height}` : 'Not available'; + const tags = video?.tags ?? []; return ( @@ -111,6 +112,27 @@ export const VideoInfoPanel: React.FC = ({ + {tags.length > 0 && ( +
+
+ +
+
+

Tags

+
+ {tags.map((tag) => ( + + {tag} + + ))} +
+
+
+ )} +
diff --git a/frontend/src/hooks/useUserPreferences.tsx b/frontend/src/hooks/useUserPreferences.tsx index e200fa2f8..483e3866a 100644 --- a/frontend/src/hooks/useUserPreferences.tsx +++ b/frontend/src/hooks/useUserPreferences.tsx @@ -15,6 +15,7 @@ export const useUserPreferences = () => { const [preferences, setPreferences] = useState({ YOLO_model_size: 'nano', GPU_Acceleration: false, + Video_Frame_Interval: 5, }); // Query for user preferences @@ -89,6 +90,17 @@ export const useUserPreferences = () => { return updatePreference(updatedPreferences); }; + /** + * Update the video keyframe sampling interval (seconds) + */ + const updateVideoFrameInterval = async (interval: number) => { + const updatedPreferences = { + ...preferences, + Video_Frame_Interval: interval, + }; + return updatePreference(updatedPreferences); + }; + return { // Data preferences, @@ -98,6 +110,7 @@ export const useUserPreferences = () => { updatePreference, updateYoloModelSize, toggleGpuAcceleration, + updateVideoFrameInterval, // For refetching preferences after external events (e.g., Model Manager window closing) refetch: preferencesQuery.refetch, diff --git a/frontend/src/pages/SearchResults/SearchResults.tsx b/frontend/src/pages/SearchResults/SearchResults.tsx index f4bb0fac5..c58893f85 100644 --- a/frontend/src/pages/SearchResults/SearchResults.tsx +++ b/frontend/src/pages/SearchResults/SearchResults.tsx @@ -3,17 +3,27 @@ import { useDispatch, useSelector } from 'react-redux'; import { AxiosError } from 'axios'; import { ImageCard } from '@/components/Media/ImageCard'; import { MediaView } from '@/components/Media/MediaView'; -import { Image, ScoredImage } from '@/types/Media'; +import { VideoCard } from '@/components/Media/VideoCard'; +import { VideoPlayerOverlay } from '@/components/VideoPlayer/VideoPlayerOverlay'; +import { Image, ScoredImage, ScoredVideo, Video } from '@/types/Media'; import { setCurrentViewIndex, setImages } from '@/features/imageSlice'; +import { + setCurrentViewIndex as setCurrentVideoViewIndex, + setVideos, +} from '@/features/videoSlice'; import { showLoader, hideLoader } from '@/features/loaderSlice'; import { showInfoDialog } from '@/features/infoDialogSlice'; import { selectImages, selectIsImageViewOpen } from '@/features/imageSelectors'; +import { selectIsVideoViewOpen, selectVideos } from '@/features/videoSelectors'; import { usePictoQuery } from '@/hooks/useQueryExtension'; import { searchImagesByTag, semanticSearchImages, + searchVideosByTag, + semanticSearchVideos, fetchModelStatus, SemanticSearchAPIResponse, + SemanticSearchVideosAPIResponse, } from '@/api/api-functions'; import { APIResponse } from '@/types/API'; import { getErrorMessage } from '@/lib/utils'; @@ -33,6 +43,16 @@ interface SemanticSearchResult extends SemanticSearchAPIResponse { type SearchQueryResult = TagSearchResult | SemanticSearchResult; +interface VideoTagSearchResult extends APIResponse { + resultType: 'tag'; +} + +interface VideoSemanticSearchResult extends SemanticSearchVideosAPIResponse { + resultType: 'semantic'; +} + +type VideoSearchQueryResult = VideoTagSearchResult | VideoSemanticSearchResult; + const getHttpStatus = (error: unknown): number | undefined => { const axiosErr = error as AxiosError; return axiosErr?.isAxiosError ? axiosErr.response?.status : undefined; @@ -46,6 +66,8 @@ export const SearchResults = () => { const mode = searchParams.get('mode') || 'auto'; const isImageViewOpen = useSelector(selectIsImageViewOpen); const displayImages = useSelector(selectImages); + const isVideoViewOpen = useSelector(selectIsVideoViewOpen); + const displayVideos = useSelector(selectVideos); const [searchError, setSearchError] = useState(null); @@ -95,6 +117,48 @@ export const SearchResults = () => { enabled: !!query, }); + // Videos run as their own query: they share the mode logic but a video + // failure (e.g. no frames embedded yet) must not blank the image results. + const { data: videoData, isSuccess: isVideoSuccess } = usePictoQuery({ + queryKey: ['search-results-videos', query, mode], + queryFn: async (): Promise => { + if (mode === 'semantic') { + const res = await semanticSearchVideos({ query }); + return { ...res, resultType: 'semantic' }; + } + + const tagResponse = await searchVideosByTag({ tag: query }); + if (mode === 'tag' || (tagResponse.data?.length ?? 0) > 0) { + return { ...tagResponse, resultType: 'tag' }; + } + + const statusRes = await fetchModelStatus(); + const semAvailable = + statusRes.success && statusRes.data + ? isSemanticSearchAvailable(statusRes.data) + : false; + + if (semAvailable) { + const semResponse = await semanticSearchVideos({ query }); + return { ...semResponse, resultType: 'semantic' }; + } + + return { ...tagResponse, resultType: 'tag' }; + }, + enabled: !!query, + }); + + useEffect(() => { + if (!isVideoSuccess || !videoData) return; + + const fetchedVideos: Video[] = + videoData.resultType === 'semantic' + ? ((videoData.data?.videos ?? []) as ScoredVideo[]) + : ((videoData.data ?? []) as Video[]); + + dispatch(setVideos(fetchedVideos)); + }, [videoData, isVideoSuccess, dispatch]); + const effectiveMode = data?.resultType || mode; useEffect(() => { @@ -188,31 +252,55 @@ export const SearchResults = () => {

Please enter a search term to find images.

- ) : displayImages.length === 0 && isSuccess ? ( + ) : displayImages.length === 0 && + displayVideos.length === 0 && + isSuccess ? (
{effectiveMode === 'semantic' ? ( -

No matches found. Try describing the photo differently.

+

No matches found. Try describing it differently.

) : ( -

No images found matching your search.

+

No photos or videos found matching your search.

)}
) : ( -
- {displayImages.map((image, index) => ( -
- dispatch(setCurrentViewIndex(index))} - /> + <> + {displayImages.length > 0 && ( +
+ {displayImages.map((image, index) => ( +
+ dispatch(setCurrentViewIndex(index))} + /> +
+ ))}
- ))} -
+ )} + + {displayVideos.length > 0 && ( + <> +

Videos

+
+ {displayVideos.map((video, index) => ( +
+ dispatch(setCurrentVideoViewIndex(index))} + /> +
+ ))} +
+ + )} + )} - {/* Media Viewer Modal */} + {/* Media Viewer Modals */} {isImageViewOpen && } + {isVideoViewOpen && }
); }; diff --git a/frontend/src/pages/SettingsPage/components/UserPreferencesCard.tsx b/frontend/src/pages/SettingsPage/components/UserPreferencesCard.tsx index 44d25cb80..2822163f9 100644 --- a/frontend/src/pages/SettingsPage/components/UserPreferencesCard.tsx +++ b/frontend/src/pages/SettingsPage/components/UserPreferencesCard.tsx @@ -1,5 +1,12 @@ import React, { useEffect, useCallback, useState } from 'react'; -import { Cpu, ChevronDown, Zap } from 'lucide-react'; +import { + Cpu, + ChevronDown, + Zap, + Trash2, + Clapperboard, + HardDrive, +} from 'lucide-react'; import { Label } from '@/components/ui/label'; import { Switch } from '@/components/ui/switch'; @@ -17,8 +24,9 @@ import { listen } from '@tauri-apps/api/event'; import { getCurrentWindow } from '@tauri-apps/api/window'; import { useUserPreferences } from '@/hooks/useUserPreferences'; +import { purgeVideoFrameCache } from '@/api/api-functions'; import SettingsCard from './SettingsCard'; -import { formatTierLabel } from '@/lib/utils'; +import { cn, formatTierLabel } from '@/lib/utils'; import { BACKEND_URL } from '@/config/Backend'; import { getInstalledModelTiers, @@ -30,12 +38,37 @@ import { /** * Component for managing user preferences in settings */ +// Coarse enough to be a meaningful cost tradeoff, fine enough to matter. +const FRAME_INTERVAL_OPTIONS = [2, 5, 10, 30]; + const UserPreferencesCard: React.FC = () => { - const { preferences, updateYoloModelSize, toggleGpuAcceleration, refetch } = - useUserPreferences(); + const { + preferences, + updateYoloModelSize, + toggleGpuAcceleration, + updateVideoFrameInterval, + refetch, + } = useUserPreferences(); const [installedTiers, setInstalledTiers] = useState([]); const [loadingTiers, setLoadingTiers] = useState(true); const [tierFetchError, setTierFetchError] = useState(null); + const [purgeState, setPurgeState] = useState<'idle' | 'purging' | 'done'>( + 'idle', + ); + // Collapsed by default: video tagging is a niche setting, so it stays out + // of the way until a user with videos goes looking for it. + const [videoSettingsOpen, setVideoSettingsOpen] = useState(false); + + const handlePurgeFrameCache = useCallback(async () => { + setPurgeState('purging'); + try { + await purgeVideoFrameCache(); + setPurgeState('done'); + } catch (err) { + console.error('Failed to purge video frame cache', err); + setPurgeState('idle'); + } + }, []); useEffect(() => { const controller = new AbortController(); @@ -219,6 +252,114 @@ const UserPreferencesCard: React.FC = () => { />
+ + {/* Video Tagging: a collapsible group so these niche controls don't + clutter the panel for users who only have photos. */} +
+ + + {videoSettingsOpen && ( +
+ {/* Video Keyframe Interval Setting */} +
+
+ +

+ How often a frame is sampled from a video for tagging. + Shorter intervals catch more detail but take longer to + process. Applies to videos tagged from now on. +

+
+ + + + + + {FRAME_INTERVAL_OPTIONS.map((seconds) => ( + + updateVideoFrameInterval(seconds).catch(console.warn) + } + > + {seconds} seconds + + ))} + + +
+ + {/* Video Frame Cache */} +
+
+ +

+ Reclaim the disk space used by sampled video frames. + Existing tags and video search keep working. +

+
+
+ + +
+
+
+ )} +
); diff --git a/frontend/src/pages/__tests__/SearchResults.test.tsx b/frontend/src/pages/__tests__/SearchResults.test.tsx index fa85ae00d..c05115f3e 100644 --- a/frontend/src/pages/__tests__/SearchResults.test.tsx +++ b/frontend/src/pages/__tests__/SearchResults.test.tsx @@ -1,6 +1,6 @@ import { render, screen, waitFor } from '@/test-utils'; import { SearchResults } from '../SearchResults/SearchResults'; -import { searchImagesByTag } from '@/api/api-functions'; +import { searchImagesByTag, searchVideosByTag } from '@/api/api-functions'; // Mock the API functions. fetchModelStatus must resolve to a well-formed // response even in tests that don't care about it -- the "no tag matches" @@ -8,12 +8,23 @@ import { searchImagesByTag } from '@/api/api-functions'; jest.mock('@/api/api-functions', () => ({ searchImagesByTag: jest.fn(), semanticSearchImages: jest.fn(), + searchVideosByTag: jest.fn(), + semanticSearchVideos: jest.fn(), fetchModelStatus: jest.fn().mockResolvedValue({ success: true, data: {} }), })); +jest.mock('@tauri-apps/api/core', () => ({ + convertFileSrc: (path: string) => path, +})); + describe('SearchResults Page', () => { beforeEach(() => { jest.clearAllMocks(); + // Videos default to no matches; the tests that care override this. + (searchVideosByTag as jest.Mock).mockResolvedValue({ + success: true, + data: [], + }); }); const renderWithQuery = (queryValue: string) => { @@ -67,7 +78,7 @@ describe('SearchResults Page', () => { // Wait for images to render await waitFor(() => { expect( - screen.queryByText(/No images found matching your search/i), + screen.queryByText(/No photos or videos found matching your search/i), ).not.toBeInTheDocument(); expect( screen.queryByText(/Please enter a search term/i), @@ -75,7 +86,36 @@ describe('SearchResults Page', () => { }); }); - test('renders no images found state when API returns empty array', async () => { + test('renders matching videos in their own section', async () => { + (searchImagesByTag as jest.Mock).mockResolvedValue({ + success: true, + data: [], + }); + (searchVideosByTag as jest.Mock).mockResolvedValue({ + success: true, + data: [ + { + id: 'v1', + path: '/clip.mp4', + thumbnailPath: '/clip-thumb.jpg', + folder_id: '1', + tags: ['beach'], + metadata: { name: 'clip.mp4' }, + }, + ], + }); + + renderWithQuery('beach'); + + expect(searchVideosByTag).toHaveBeenCalledWith({ tag: 'beach' }); + + await waitFor(() => { + expect(screen.getByText('Videos')).toBeInTheDocument(); + expect(screen.getByLabelText(/Play clip.mp4/i)).toBeInTheDocument(); + }); + }); + + test('renders no results state when both APIs return empty arrays', async () => { (searchImagesByTag as jest.Mock).mockResolvedValue({ success: true, data: [], @@ -85,7 +125,7 @@ describe('SearchResults Page', () => { await waitFor(() => { expect( - screen.getByText(/No images found matching your search/i), + screen.getByText(/No photos or videos found matching your search/i), ).toBeInTheDocument(); }); }); diff --git a/frontend/src/types/Media.ts b/frontend/src/types/Media.ts index 57f8b3a2b..bfc022f97 100644 --- a/frontend/src/types/Media.ts +++ b/frontend/src/types/Media.ts @@ -48,6 +48,12 @@ export interface Video { tags?: string[]; } +export interface ScoredVideo extends Video { + score: number; + /** Timestamp (seconds) of the keyframe that matched best. */ + best_frame_timestamp?: number | null; +} + export interface ImageGalleryProps { mediaItems: Image[]; title?: string; From fab7ef08291dbb6d7b7cce218bbc9529d8f146c6 Mon Sep 17 00:00:00 2001 From: ROHAN PANDEY <95585299+rohan-pandeyy@users.noreply.github.com> Date: Fri, 24 Jul 2026 11:21:27 +0530 Subject: [PATCH 05/10] fix: count videos in folder details so a video-only folder isn't "empty" db_get_all_folder_details counted only images, so a folder containing only videos reported image_count 0 and the settings card showed "Folder is empty", hiding the tagging progress UI. Add a video_count (via COUNT(DISTINCT ...) to avoid the join multiplying rows) and treat a folder as non-empty when it has images or videos. --- backend/app/database/folders.py | 23 +++++++++++-------- backend/app/routes/folders.py | 2 ++ backend/app/schemas/folders.py | 1 + backend/tests/test_folders.py | 2 ++ .../components/FolderManagementCard.tsx | 2 +- frontend/src/types/Folder.ts | 1 + 6 files changed, 21 insertions(+), 10 deletions(-) diff --git a/backend/app/database/folders.py b/backend/app/database/folders.py index 19ef1fa77..e4081b92f 100644 --- a/backend/app/database/folders.py +++ b/backend/app/database/folders.py @@ -402,29 +402,34 @@ def db_get_folder_ids_by_paths( def db_get_all_folder_details() -> ( - List[Tuple[str, str, Optional[str], int, bool, Optional[bool], int]] + List[Tuple[str, str, Optional[str], int, bool, Optional[bool], int, int]] ): """ Get all folder details including folder_id, folder_path, parent_folder_id, - last_modified_time, AI_Tagging, taggingCompleted, and image_count. + last_modified_time, AI_Tagging, taggingCompleted, image_count and + video_count. Returns list of tuples with all folder information. """ conn = sqlite3.connect(DATABASE_PATH) cursor = conn.cursor() try: + # COUNT(DISTINCT ...) because joining both media tables multiplies the + # rows: a folder with 3 images and 4 videos yields 12 join rows. cursor.execute( """ - SELECT - f.folder_id, - f.folder_path, - f.parent_folder_id, - f.last_modified_time, - f.AI_Tagging, + SELECT + f.folder_id, + f.folder_path, + f.parent_folder_id, + f.last_modified_time, + f.AI_Tagging, f.taggingCompleted, - COUNT(i.id) as image_count + COUNT(DISTINCT i.id) as image_count, + COUNT(DISTINCT v.id) as video_count FROM folders f LEFT JOIN images i ON f.folder_id = i.folder_id + LEFT JOIN videos v ON f.folder_id = v.folder_id GROUP BY f.folder_id ORDER BY f.folder_path """ diff --git a/backend/app/routes/folders.py b/backend/app/routes/folders.py index 537ea262c..3763e88ec 100644 --- a/backend/app/routes/folders.py +++ b/backend/app/routes/folders.py @@ -475,6 +475,7 @@ def get_all_folders(): ai_tagging, tagging_completed, image_count, + video_count, ) = folder_data folders.append( FolderDetails( @@ -485,6 +486,7 @@ def get_all_folders(): AI_Tagging=ai_tagging, taggingCompleted=tagging_completed, image_count=image_count, + video_count=video_count, ) ) diff --git a/backend/app/schemas/folders.py b/backend/app/schemas/folders.py index 749f51657..cea3ddcf9 100644 --- a/backend/app/schemas/folders.py +++ b/backend/app/schemas/folders.py @@ -31,6 +31,7 @@ class FolderDetails(BaseModel): AI_Tagging: bool taggingCompleted: Optional[bool] = None image_count: int = 0 + video_count: int = 0 class GetAllFoldersData(BaseModel): diff --git a/backend/tests/test_folders.py b/backend/tests/test_folders.py index 31f6b1cd4..dd16ff19b 100644 --- a/backend/tests/test_folders.py +++ b/backend/tests/test_folders.py @@ -99,6 +99,7 @@ def sample_folder_details(): True, # AI_Tagging False, # taggingCompleted 0, # image_count + 7, # video_count -- a video-only folder is not "empty" ), ( "folder-id-2", @@ -108,6 +109,7 @@ def sample_folder_details(): False, True, 25, + 0, ), ] diff --git a/frontend/src/pages/SettingsPage/components/FolderManagementCard.tsx b/frontend/src/pages/SettingsPage/components/FolderManagementCard.tsx index d439dbb08..4b00563a2 100644 --- a/frontend/src/pages/SettingsPage/components/FolderManagementCard.tsx +++ b/frontend/src/pages/SettingsPage/components/FolderManagementCard.tsx @@ -92,7 +92,7 @@ const FolderManagementCard: React.FC = () => { {folder.AI_Tagging && (
- {!folder.image_count ? ( + {!folder.image_count && !folder.video_count ? (
Folder is empty
diff --git a/frontend/src/types/Folder.ts b/frontend/src/types/Folder.ts index 49657c91b..2eeea186c 100644 --- a/frontend/src/types/Folder.ts +++ b/frontend/src/types/Folder.ts @@ -6,6 +6,7 @@ export interface FolderDetails { AI_Tagging: boolean; taggingCompleted?: boolean; image_count?: number; + video_count?: number; } export interface GetAllFoldersData { From e18624a2d1d25a786f636c07914ec2153700550a Mon Sep 17 00:00:00 2001 From: ROHAN PANDEY <95585299+rohan-pandeyy@users.noreply.github.com> Date: Fri, 24 Jul 2026 11:21:57 +0530 Subject: [PATCH 06/10] feat: include videos in tagging-progress bar and background alert The sync-microservice /status endpoint counted only images, so the per-folder progress bar and the app-wide processing alert never moved for videos. Add video counts to the status query (percentages now span images and videos combined) and aggregate them on the frontend. A video counts as embedded once tagged with no unembedded frames left, so an undecodable video never stalls the bar. Alert copy updated to say "photos and videos". --- .../LibraryProcessingIndicator.tsx | 16 ++--- .../src/hooks/useLibraryProcessingStatus.ts | 46 +++++++------- frontend/src/types/FolderStatus.ts | 3 + sync-microservice/app/database/folders.py | 60 +++++++++++++++---- sync-microservice/app/routes/folders.py | 3 + sync-microservice/app/schemas/folders.py | 13 +++- 6 files changed, 98 insertions(+), 43 deletions(-) diff --git a/frontend/src/components/BackgroundTasks/LibraryProcessingIndicator.tsx b/frontend/src/components/BackgroundTasks/LibraryProcessingIndicator.tsx index 6896a5472..a8786cb7e 100644 --- a/frontend/src/components/BackgroundTasks/LibraryProcessingIndicator.tsx +++ b/frontend/src/components/BackgroundTasks/LibraryProcessingIndicator.tsx @@ -9,7 +9,7 @@ import { ROUTES } from '@/constants/routes'; interface Dismissal { phase: LibraryProcessingPhase; - totalImages: number; + totalItems: number; } /** @@ -17,17 +17,17 @@ interface Dismissal { * Hidden on Settings, which has its own bars. Needs router context. */ export const LibraryProcessingIndicator: React.FC = () => { - const { phase, percentage, totalImages, semanticAvailable } = + const { phase, percentage, totalItems, semanticAvailable } = useLibraryProcessingStatus(); const location = useLocation(); // Dismissal is per-phase and invalidated when the library grows, so the - // alert resurfaces when indexing starts or new photos appear. + // alert resurfaces when indexing starts or new media appears. const [dismissal, setDismissal] = useState(null); const dismissed = dismissal !== null && dismissal.phase === phase && - totalImages <= dismissal.totalImages; + totalItems <= dismissal.totalItems; const onSettingsPage = location.pathname.includes(`/${ROUTES.SETTINGS}`); @@ -36,15 +36,15 @@ export const LibraryProcessingIndicator: React.FC = () => { const step = (n: number) => (semanticAvailable ? `Step ${n} of 2: ` : ''); const description = phase === 'tagging' - ? `${step(1)}Looking through your photos to recognize what's in them.` - : 'Step 2 of 2: Making your photos searchable, so you can find them by describing them in your own words.'; + ? `${step(1)}Looking through your photos and videos to recognize what's in them.` + : 'Step 2 of 2: Making your photos and videos searchable, so you can find them by describing them in your own words.'; return ( setDismissal({ phase, totalImages })} + onDismiss={() => setDismissal({ phase, totalItems })} /> ); }; diff --git a/frontend/src/hooks/useLibraryProcessingStatus.ts b/frontend/src/hooks/useLibraryProcessingStatus.ts index d6baf30bc..2ac8ce8b2 100644 --- a/frontend/src/hooks/useLibraryProcessingStatus.ts +++ b/frontend/src/hooks/useLibraryProcessingStatus.ts @@ -14,9 +14,10 @@ export interface LibraryProcessingStatus { phase: LibraryProcessingPhase; /** Progress (0-100) of the current phase; 100 when idle */ percentage: number; - totalImages: number; - taggedImages: number; - embeddedImages: number; + /** Counts are over images and videos combined. */ + totalItems: number; + taggedItems: number; + embeddedItems: number; } const aggregate = (data: APIResponse | undefined) => { @@ -25,17 +26,18 @@ const aggregate = (data: APIResponse | undefined) => { ? (rawFolders as FolderTaggingInfo[]) : []; - let totalImages = 0; - let taggedImages = 0; - let embeddedImages = 0; + let totalItems = 0; + let taggedItems = 0; + let embeddedItems = 0; for (const folder of folders) { // Only AI-tagging folders are ever processed if (!folder?.ai_tagging) continue; - totalImages += folder.total_images ?? 0; - taggedImages += folder.tagged_images ?? 0; - embeddedImages += folder.embedded_images ?? 0; + totalItems += (folder.total_images ?? 0) + (folder.total_videos ?? 0); + taggedItems += (folder.tagged_images ?? 0) + (folder.tagged_videos ?? 0); + embeddedItems += + (folder.embedded_images ?? 0) + (folder.embedded_videos ?? 0); } - return { totalImages, taggedImages, embeddedImages }; + return { totalItems, taggedItems, embeddedItems }; }; /** @@ -59,12 +61,12 @@ export const useLibraryProcessingStatus = (): LibraryProcessingStatus => { queryFn: getFoldersTaggingStatus, staleTime: 1000, refetchInterval: (query) => { - const { totalImages, taggedImages, embeddedImages } = aggregate( + const { totalItems, taggedItems, embeddedItems } = aggregate( query.state.data, ); const busy = - taggedImages < totalImages || - (semanticAvailable && embeddedImages < totalImages); + taggedItems < totalItems || + (semanticAvailable && embeddedItems < totalItems); return busy ? 1000 : 10000; }, refetchIntervalInBackground: true, @@ -73,30 +75,30 @@ export const useLibraryProcessingStatus = (): LibraryProcessingStatus => { refetchOnWindowFocus: false, }); - const { totalImages, taggedImages, embeddedImages } = aggregate( + const { totalItems, taggedItems, embeddedItems } = aggregate( taggingStatusQuery.data, ); let phase: LibraryProcessingPhase = 'idle'; let percentage = 100; - if (totalImages > 0 && taggedImages < totalImages) { + if (totalItems > 0 && taggedItems < totalItems) { phase = 'tagging'; - percentage = (taggedImages / totalImages) * 100; + percentage = (taggedItems / totalItems) * 100; } else if ( semanticAvailable && - totalImages > 0 && - embeddedImages < totalImages + totalItems > 0 && + embeddedItems < totalItems ) { phase = 'indexing'; - percentage = (embeddedImages / totalImages) * 100; + percentage = (embeddedItems / totalItems) * 100; } return { semanticAvailable, phase, percentage, - totalImages, - taggedImages, - embeddedImages, + totalItems, + taggedItems, + embeddedItems, }; }; diff --git a/frontend/src/types/FolderStatus.ts b/frontend/src/types/FolderStatus.ts index cf68ad3e0..950f6ef6a 100644 --- a/frontend/src/types/FolderStatus.ts +++ b/frontend/src/types/FolderStatus.ts @@ -6,6 +6,9 @@ export interface FolderTaggingInfo { total_images: number; tagged_images: number; embedded_images: number; + total_videos: number; + tagged_videos: number; + embedded_videos: number; ai_tagging: boolean; } diff --git a/sync-microservice/app/database/folders.py b/sync-microservice/app/database/folders.py index 22429992a..45995c30a 100644 --- a/sync-microservice/app/database/folders.py +++ b/sync-microservice/app/database/folders.py @@ -12,7 +12,12 @@ class FolderTaggingInfo(NamedTuple): - """Represents folder tagging and semantic-embedding information""" + """Represents folder tagging and semantic-embedding information. + + Percentages are over images and videos combined, so a video-only folder + still reports real progress. The raw per-medium counts are exposed so the + frontend can aggregate them however it needs. + """ folder_id: FolderId folder_path: FolderPath @@ -21,6 +26,9 @@ class FolderTaggingInfo(NamedTuple): total_images: int tagged_images: int embedded_images: int + total_videos: int + tagged_videos: int + embedded_videos: int ai_tagging: bool @@ -80,27 +88,48 @@ def db_check_database_connection() -> bool: def db_get_tagging_progress() -> List[FolderTaggingInfo]: """ Calculate tagging and semantic-embedding percentages for all folders. - Each percentage = (processed images / total images) * 100. + Each percentage = (processed media / total media) * 100, counting images + and videos together. Returns: - List of FolderTaggingInfo with percentages, raw counts, and each - folder's AI_Tagging flag + List of FolderTaggingInfo with percentages, raw per-medium counts, and + each folder's AI_Tagging flag """ conn = sqlite3.connect(DATABASE_PATH) cursor = conn.cursor() try: + # COUNT(DISTINCT ...) because joining both media tables multiplies the + # rows. embedded_videos is a correlated subquery: a video is "embedded" + # once tagged with no unembedded frames left -- a zero-frame + # (undecodable) video counts as embedded so it never stalls the bar. cursor.execute( """ SELECT f.folder_id, f.folder_path, f.AI_Tagging, - COUNT(i.id) as total_images, - COUNT(CASE WHEN i.isTagged = 1 THEN 1 END) as tagged_images, - COUNT(CASE WHEN i.isEmbedded = 1 THEN 1 END) as embedded_images + COUNT(DISTINCT i.id) as total_images, + COUNT(DISTINCT CASE WHEN i.isTagged = 1 THEN i.id END) + as tagged_images, + COUNT(DISTINCT CASE WHEN i.isEmbedded = 1 THEN i.id END) + as embedded_images, + COUNT(DISTINCT v.id) as total_videos, + COUNT(DISTINCT CASE WHEN v.isTagged = 1 THEN v.id END) + as tagged_videos, + ( + SELECT COUNT(*) + FROM videos v2 + WHERE v2.folder_id = f.folder_id + AND v2.isTagged = 1 + AND NOT EXISTS ( + SELECT 1 FROM video_frames vf + WHERE vf.video_id = v2.id AND vf.isEmbedded = 0 + ) + ) as embedded_videos FROM folders f LEFT JOIN images i ON f.folder_id = i.folder_id + LEFT JOIN videos v ON f.folder_id = v.folder_id GROUP BY f.folder_id, f.folder_path, f.AI_Tagging """ ) @@ -115,11 +144,17 @@ def db_get_tagging_progress() -> List[FolderTaggingInfo]: total_images, tagged_images, embedded_images, + total_videos, + tagged_videos, + embedded_videos, ) in results: - # Calculate percentages, handle division by zero - if total_images > 0: - tagging_percentage = (tagged_images / total_images) * 100 - embedding_percentage = (embedded_images / total_images) * 100 + # Percentages are over images and videos together. + total_media = total_images + total_videos + if total_media > 0: + tagging_percentage = (tagged_images + tagged_videos) / total_media * 100 + embedding_percentage = ( + (embedded_images + embedded_videos) / total_media * 100 + ) else: tagging_percentage = 0.0 embedding_percentage = 0.0 @@ -133,6 +168,9 @@ def db_get_tagging_progress() -> List[FolderTaggingInfo]: total_images=total_images, tagged_images=tagged_images, embedded_images=embedded_images, + total_videos=total_videos, + tagged_videos=tagged_videos, + embedded_videos=embedded_videos, ai_tagging=bool(ai_tagging), ) ) diff --git a/sync-microservice/app/routes/folders.py b/sync-microservice/app/routes/folders.py index 115458935..b2285ea2d 100644 --- a/sync-microservice/app/routes/folders.py +++ b/sync-microservice/app/routes/folders.py @@ -36,6 +36,9 @@ def get_folders_tagging_status(): total_images=folder.total_images, tagged_images=folder.tagged_images, embedded_images=folder.embedded_images, + total_videos=folder.total_videos, + tagged_videos=folder.tagged_videos, + embedded_videos=folder.embedded_videos, ai_tagging=folder.ai_tagging, ) for folder in tagging_progress diff --git a/sync-microservice/app/schemas/folders.py b/sync-microservice/app/schemas/folders.py index d3aefdf9a..48ca894be 100644 --- a/sync-microservice/app/schemas/folders.py +++ b/sync-microservice/app/schemas/folders.py @@ -11,13 +11,13 @@ class FolderTaggingInfo(BaseModel): ..., ge=0, le=100, - description="Percentage of images that have been tagged (0-100)", + description="Percentage of images and videos tagged (0-100)", ) embedding_percentage: float = Field( ..., ge=0, le=100, - description="Percentage of images with semantic embeddings (0-100)", + description="Percentage of images and videos with embeddings (0-100)", ) total_images: int = Field( ..., ge=0, description="Total number of images in the folder" @@ -28,6 +28,15 @@ class FolderTaggingInfo(BaseModel): embedded_images: int = Field( ..., ge=0, description="Number of images with semantic embeddings" ) + total_videos: int = Field( + ..., ge=0, description="Total number of videos in the folder" + ) + tagged_videos: int = Field( + ..., ge=0, description="Number of videos that have been tagged" + ) + embedded_videos: int = Field( + ..., ge=0, description="Number of videos whose frames are embedded" + ) ai_tagging: bool = Field( ..., description="Whether AI tagging is enabled for the folder" ) From 23ee9504d46ed975d84341b81f78d3e46cd5b1b4 Mon Sep 17 00:00:00 2001 From: ROHAN PANDEY <95585299+rohan-pandeyy@users.noreply.github.com> Date: Sat, 25 Jul 2026 02:52:42 +0530 Subject: [PATCH 07/10] fix: address review feedback on the video tagging PR - db_get_video_tags: chunk the video-id IN() clause below SQLite's 999 placeholder limit, so libraries with >999 videos keep their tags instead of erroring out to an empty mapping. - user preferences: default and validate Video_Frame_Interval against the sampler's VIDEO_FRAME_INTERVAL_SECONDS and its 0.5-300 range, so the API reports the same interval the sampler actually uses. - VideoInfoPanel: key tag chips by `${tag}-${index}` to avoid duplicate React keys on repeated labels. - SearchResults: track image and video search errors independently so an error in one medium shows inline without hiding the other's results; reuse the already-fetched model status and log video-query failures. - sync progress query: pre-aggregate images and videos per folder before joining, removing the images x videos cartesian fan-out. --- backend/app/database/video_frames.py | 43 ++++-- backend/app/routes/user_preferences.py | 9 +- backend/app/schemas/user_preferences.py | 17 +- docs/backend/backend_python/openapi.json | 8 +- .../components/VideoPlayer/VideoInfoPanel.tsx | 4 +- .../src/pages/SearchResults/SearchResults.tsx | 145 ++++++++++++------ .../pages/__tests__/SearchResults.test.tsx | 42 ++++- sync-microservice/app/database/folders.py | 58 ++++--- 8 files changed, 228 insertions(+), 98 deletions(-) diff --git a/backend/app/database/video_frames.py b/backend/app/database/video_frames.py index 98a9c1fe2..eee36ed96 100644 --- a/backend/app/database/video_frames.py +++ b/backend/app/database/video_frames.py @@ -444,30 +444,41 @@ def db_get_all_frame_embeddings( def db_get_video_tags(video_ids: Optional[List[str]] = None) -> Dict[str, List[str]]: """Display tag names per video. Pass video_ids to restrict, or None for every video.""" + base_query = """ + SELECT vc.video_id, m.name + FROM video_classes_display vc + JOIN mappings m ON vc.class_id = m.class_id + """ + order_by = " ORDER BY vc.video_id, m.name" + conn = None try: conn = _connect() cursor = conn.cursor() - query = """ - SELECT vc.video_id, m.name - FROM video_classes_display vc - JOIN mappings m ON vc.class_id = m.class_id - """ - params: List[str] = [] - if video_ids is not None: + tags: Dict[str, List[str]] = {} + + def collect(query: str, params: List[str]) -> None: + cursor.execute(query, params) + for video_id, name in cursor.fetchall(): + tags.setdefault(video_id, []).append(name) + + if video_ids is None: + collect(base_query + order_by, []) + else: if not video_ids: return {} - placeholders = ",".join("?" for _ in video_ids) - query += f" WHERE vc.video_id IN ({placeholders})" - params = video_ids - query += " ORDER BY vc.video_id, m.name" - - cursor.execute(query, params) + # Chunk below SQLite's 999-placeholder limit so libraries with + # more than ~999 videos don't error out and lose every tag. + chunk_size = 500 + for i in range(0, len(video_ids), chunk_size): + chunk = video_ids[i : i + chunk_size] + placeholders = ",".join("?" for _ in chunk) + collect( + base_query + f" WHERE vc.video_id IN ({placeholders})" + order_by, + chunk, + ) - tags: Dict[str, List[str]] = {} - for video_id, name in cursor.fetchall(): - tags.setdefault(video_id, []).append(name) return tags except sqlite3.Error as e: logger.error(f"Error getting video tags: {e}") diff --git a/backend/app/routes/user_preferences.py b/backend/app/routes/user_preferences.py index 229d121ce..982654613 100644 --- a/backend/app/routes/user_preferences.py +++ b/backend/app/routes/user_preferences.py @@ -1,4 +1,5 @@ from fastapi import APIRouter, HTTPException, status +from app.config.settings import VIDEO_FRAME_INTERVAL_SECONDS from app.database.metadata import db_get_metadata, db_update_metadata from app.schemas.user_preferences import ( GetUserPreferencesResponse, @@ -30,7 +31,9 @@ def get_user_preferences(): user_preferences = UserPreferencesData( YOLO_model_size=user_prefs_data.get("YOLO_model_size", "small"), GPU_Acceleration=user_prefs_data.get("GPU_Acceleration", True), - Video_Frame_Interval=user_prefs_data.get("Video_Frame_Interval", 5.0), + Video_Frame_Interval=user_prefs_data.get( + "Video_Frame_Interval", VIDEO_FRAME_INTERVAL_SECONDS + ), ) return GetUserPreferencesResponse( @@ -102,7 +105,9 @@ def update_user_preferences(request: UpdateUserPreferencesRequest): user_preferences = UserPreferencesData( YOLO_model_size=current_user_prefs.get("YOLO_model_size", "small"), GPU_Acceleration=current_user_prefs.get("GPU_Acceleration", True), - Video_Frame_Interval=current_user_prefs.get("Video_Frame_Interval", 5.0), + Video_Frame_Interval=current_user_prefs.get( + "Video_Frame_Interval", VIDEO_FRAME_INTERVAL_SECONDS + ), ) return UpdateUserPreferencesResponse( diff --git a/backend/app/schemas/user_preferences.py b/backend/app/schemas/user_preferences.py index 094698e85..73e0fb47b 100644 --- a/backend/app/schemas/user_preferences.py +++ b/backend/app/schemas/user_preferences.py @@ -1,6 +1,13 @@ from pydantic import BaseModel, Field from typing import Optional, Literal +from app.config.settings import VIDEO_FRAME_INTERVAL_SECONDS + +# Kept in lockstep with the sampler's config (settings.py) so the API reports, +# validates, and defaults to exactly what video_util_get_frame_interval() uses. +VIDEO_FRAME_INTERVAL_MIN = 0.5 +VIDEO_FRAME_INTERVAL_MAX = 300.0 + class UserPreferencesData(BaseModel): """User preferences data structure""" @@ -9,7 +16,11 @@ class UserPreferencesData(BaseModel): GPU_Acceleration: bool = True # Seconds between sampled video keyframes. Lower means finer tag coverage # and proportionally more inference per video. - Video_Frame_Interval: float = Field(default=5.0, ge=1.0, le=60.0) + Video_Frame_Interval: float = Field( + default=VIDEO_FRAME_INTERVAL_SECONDS, + ge=VIDEO_FRAME_INTERVAL_MIN, + le=VIDEO_FRAME_INTERVAL_MAX, + ) class GetUserPreferencesResponse(BaseModel): @@ -25,7 +36,9 @@ class UpdateUserPreferencesRequest(BaseModel): YOLO_model_size: Optional[Literal["nano", "small", "medium"]] = None GPU_Acceleration: Optional[bool] = None - Video_Frame_Interval: Optional[float] = Field(default=None, ge=1.0, le=60.0) + Video_Frame_Interval: Optional[float] = Field( + default=None, ge=VIDEO_FRAME_INTERVAL_MIN, le=VIDEO_FRAME_INTERVAL_MAX + ) class UpdateUserPreferencesResponse(BaseModel): diff --git a/docs/backend/backend_python/openapi.json b/docs/backend/backend_python/openapi.json index 62af6bd15..7aa2ade9d 100644 --- a/docs/backend/backend_python/openapi.json +++ b/docs/backend/backend_python/openapi.json @@ -4068,8 +4068,8 @@ "anyOf": [ { "type": "number", - "maximum": 60.0, - "minimum": 1.0 + "maximum": 300.0, + "minimum": 0.5 }, { "type": "null" @@ -4124,8 +4124,8 @@ }, "Video_Frame_Interval": { "type": "number", - "maximum": 60.0, - "minimum": 1.0, + "maximum": 300.0, + "minimum": 0.5, "title": "Video Frame Interval", "default": 5.0 } diff --git a/frontend/src/components/VideoPlayer/VideoInfoPanel.tsx b/frontend/src/components/VideoPlayer/VideoInfoPanel.tsx index 9f248ab74..4bc67729e 100644 --- a/frontend/src/components/VideoPlayer/VideoInfoPanel.tsx +++ b/frontend/src/components/VideoPlayer/VideoInfoPanel.tsx @@ -120,9 +120,9 @@ export const VideoInfoPanel: React.FC = ({

Tags

- {tags.map((tag) => ( + {tags.map((tag, index) => ( {tag} diff --git a/frontend/src/pages/SearchResults/SearchResults.tsx b/frontend/src/pages/SearchResults/SearchResults.tsx index c58893f85..a97a8c605 100644 --- a/frontend/src/pages/SearchResults/SearchResults.tsx +++ b/frontend/src/pages/SearchResults/SearchResults.tsx @@ -118,8 +118,15 @@ export const SearchResults = () => { }); // Videos run as their own query: they share the mode logic but a video - // failure (e.g. no frames embedded yet) must not blank the image results. - const { data: videoData, isSuccess: isVideoSuccess } = usePictoQuery({ + // failure (e.g. no frames embedded yet) must not blank the image results, + // and vice versa. + const { + data: videoData, + isSuccess: isVideoSuccess, + isError: isVideoError, + error: videoError, + errorMessage: videoErrorMessage, + } = usePictoQuery({ queryKey: ['search-results-videos', query, mode], queryFn: async (): Promise => { if (mode === 'semantic') { @@ -132,11 +139,16 @@ export const SearchResults = () => { return { ...tagResponse, resultType: 'tag' }; } - const statusRes = await fetchModelStatus(); - const semAvailable = - statusRes.success && statusRes.data - ? isSemanticSearchAvailable(statusRes.data) - : false; + // Reuse the status already fetched above; only fetch again if it + // hasn't resolved yet, avoiding a redundant round-trip per search. + let semAvailable = semanticAvailable; + if (!isStatusSuccess) { + const statusRes = await fetchModelStatus(); + semAvailable = + statusRes.success && statusRes.data + ? isSemanticSearchAvailable(statusRes.data) + : false; + } if (semAvailable) { const semResponse = await semanticSearchVideos({ query }); @@ -159,6 +171,22 @@ export const SearchResults = () => { dispatch(setVideos(fetchedVideos)); }, [videoData, isVideoSuccess, dispatch]); + // A failed video search should not be silent; surface it in the logs like + // the image query's error branch does. + useEffect(() => { + if (isVideoError) { + console.error( + 'Video search failed:', + getErrorMessage(videoError, videoErrorMessage), + ); + } + }, [isVideoError, videoError, videoErrorMessage]); + + const videoSearchError = isVideoError + ? getErrorMessage(videoError, videoErrorMessage) || + 'Failed to search videos' + : null; + const effectiveMode = data?.resultType || mode; useEffect(() => { @@ -240,7 +268,12 @@ export const SearchResults = () => {
)} - {searchError ? ( + {!query ? ( +
+

Please enter a search term to find photos and videos.

+
+ ) : searchError && videoSearchError ? ( + // Only a total failure (both media types) takes over the whole view.

@@ -248,53 +281,75 @@ export const SearchResults = () => {

{searchError}

- ) : !query ? ( -
-

Please enter a search term to find images.

-
- ) : displayImages.length === 0 && - displayVideos.length === 0 && - isSuccess ? ( -
- {effectiveMode === 'semantic' ? ( -

No matches found. Try describing it differently.

- ) : ( -

No photos or videos found matching your search.

- )} -
) : ( <> - {displayImages.length > 0 && ( -
- {displayImages.map((image, index) => ( -
- dispatch(setCurrentViewIndex(index))} - /> -
- ))} + {/* Photos: an image-search error is shown inline so it never hides + successfully-fetched videos. */} + {searchError ? ( +
+ + Couldn't load photo results: {searchError}
- )} - - {displayVideos.length > 0 && ( - <> -

Videos

+ ) : ( + displayImages.length > 0 && (
- {displayVideos.map((video, index) => ( -
- ( +
+ dispatch(setCurrentVideoViewIndex(index))} + onClick={() => dispatch(setCurrentViewIndex(index))} />
))}
- + ) )} + + {/* Videos: symmetric — a video-search error is inline and never + hides successfully-fetched photos. */} + {videoSearchError ? ( +
+ + Couldn't load video results: {videoSearchError} +
+ ) : ( + displayVideos.length > 0 && ( + <> +

Videos

+
+ {displayVideos.map((video, index) => ( +
+ + dispatch(setCurrentVideoViewIndex(index)) + } + /> +
+ ))} +
+ + ) + )} + + {/* Both searches succeeded but nothing matched. */} + {!searchError && + !videoSearchError && + isSuccess && + isVideoSuccess && + displayImages.length === 0 && + displayVideos.length === 0 && ( +
+ {effectiveMode === 'semantic' ? ( +

No matches found. Try describing it differently.

+ ) : ( +

No photos or videos found matching your search.

+ )} +
+ )} )} diff --git a/frontend/src/pages/__tests__/SearchResults.test.tsx b/frontend/src/pages/__tests__/SearchResults.test.tsx index c05115f3e..2e8838ed2 100644 --- a/frontend/src/pages/__tests__/SearchResults.test.tsx +++ b/frontend/src/pages/__tests__/SearchResults.test.tsx @@ -43,7 +43,7 @@ describe('SearchResults Page', () => { // Check for empty state message expect( - screen.getByText(/Please enter a search term to find images/i), + screen.getByText(/Please enter a search term to find/i), ).toBeInTheDocument(); }); @@ -130,10 +130,48 @@ describe('SearchResults Page', () => { }); }); - test('renders error state when API request fails', async () => { + test('a photo-search error is shown inline and does not hide video results', async () => { (searchImagesByTag as jest.Mock).mockRejectedValue( new Error('Network Error'), ); + (searchVideosByTag as jest.Mock).mockResolvedValue({ + success: true, + data: [ + { + id: 'v1', + path: '/clip.mp4', + thumbnailPath: '/clip-thumb.jpg', + folder_id: '1', + tags: ['beach'], + metadata: { name: 'clip.mp4' }, + }, + ], + }); + + renderWithQuery('dog'); + + await waitFor( + () => { + // Inline photo error, not the full-screen takeover... + expect( + screen.getByText(/Couldn't load photo results/i), + ).toBeInTheDocument(); + expect(screen.queryByText(/^Search Failed$/i)).not.toBeInTheDocument(); + // ...and the videos still render. + expect(screen.getByText('Videos')).toBeInTheDocument(); + expect(screen.getByLabelText(/Play clip.mp4/i)).toBeInTheDocument(); + }, + { timeout: 3000 }, + ); + }); + + test('the Search Failed takeover appears only when both searches fail', async () => { + (searchImagesByTag as jest.Mock).mockRejectedValue( + new Error('Network Error'), + ); + (searchVideosByTag as jest.Mock).mockRejectedValue( + new Error('Network Error'), + ); renderWithQuery('dog'); diff --git a/sync-microservice/app/database/folders.py b/sync-microservice/app/database/folders.py index 45995c30a..6447445ca 100644 --- a/sync-microservice/app/database/folders.py +++ b/sync-microservice/app/database/folders.py @@ -99,38 +99,46 @@ def db_get_tagging_progress() -> List[FolderTaggingInfo]: cursor = conn.cursor() try: - # COUNT(DISTINCT ...) because joining both media tables multiplies the - # rows. embedded_videos is a correlated subquery: a video is "embedded" - # once tagged with no unembedded frames left -- a zero-frame - # (undecodable) video counts as embedded so it never stalls the bar. + # Pre-aggregate each media table by folder before joining, so a folder + # with M images and N videos never fans out to M*N intermediate rows. + # A video is "embedded" once tagged with no unembedded frames left -- a + # zero-frame (undecodable) video counts as embedded so it never stalls + # the bar. cursor.execute( """ SELECT f.folder_id, f.folder_path, f.AI_Tagging, - COUNT(DISTINCT i.id) as total_images, - COUNT(DISTINCT CASE WHEN i.isTagged = 1 THEN i.id END) - as tagged_images, - COUNT(DISTINCT CASE WHEN i.isEmbedded = 1 THEN i.id END) - as embedded_images, - COUNT(DISTINCT v.id) as total_videos, - COUNT(DISTINCT CASE WHEN v.isTagged = 1 THEN v.id END) - as tagged_videos, - ( - SELECT COUNT(*) - FROM videos v2 - WHERE v2.folder_id = f.folder_id - AND v2.isTagged = 1 - AND NOT EXISTS ( - SELECT 1 FROM video_frames vf - WHERE vf.video_id = v2.id AND vf.isEmbedded = 0 - ) - ) as embedded_videos + COALESCE(img.total_images, 0) as total_images, + COALESCE(img.tagged_images, 0) as tagged_images, + COALESCE(img.embedded_images, 0) as embedded_images, + COALESCE(vid.total_videos, 0) as total_videos, + COALESCE(vid.tagged_videos, 0) as tagged_videos, + COALESCE(vid.embedded_videos, 0) as embedded_videos FROM folders f - LEFT JOIN images i ON f.folder_id = i.folder_id - LEFT JOIN videos v ON f.folder_id = v.folder_id - GROUP BY f.folder_id, f.folder_path, f.AI_Tagging + LEFT JOIN ( + SELECT folder_id, + COUNT(*) as total_images, + SUM(CASE WHEN isTagged = 1 THEN 1 ELSE 0 END) + as tagged_images, + SUM(CASE WHEN isEmbedded = 1 THEN 1 ELSE 0 END) + as embedded_images + FROM images + GROUP BY folder_id + ) img ON img.folder_id = f.folder_id + LEFT JOIN ( + SELECT v2.folder_id, + COUNT(*) as total_videos, + SUM(CASE WHEN v2.isTagged = 1 THEN 1 ELSE 0 END) + as tagged_videos, + SUM(CASE WHEN v2.isTagged = 1 AND NOT EXISTS ( + SELECT 1 FROM video_frames vf + WHERE vf.video_id = v2.id AND vf.isEmbedded = 0 + ) THEN 1 ELSE 0 END) as embedded_videos + FROM videos v2 + GROUP BY v2.folder_id + ) vid ON vid.folder_id = f.folder_id """ ) From e336518f762f37928460ffb484c1bd79d0673379 Mon Sep 17 00:00:00 2001 From: ROHAN PANDEY <95585299+rohan-pandeyy@users.noreply.github.com> Date: Sat, 25 Jul 2026 23:04:22 +0530 Subject: [PATCH 08/10] test: create the videos table in the folder-details test fixture db_get_all_folder_details now LEFT JOINs videos (for video_count), so the TestFoldersUnit fixture must create that table too. Without it the merged test_db_get_all_folder_details errored with "no such table: videos" on CI, which runs the PR merged against the latest main. Also assert the new video_count column (0 for an image-only folder). --- backend/tests/test_folders.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/tests/test_folders.py b/backend/tests/test_folders.py index 7fd9f2712..dc41f1087 100644 --- a/backend/tests/test_folders.py +++ b/backend/tests/test_folders.py @@ -36,6 +36,7 @@ db_find_parent_folder_id, ) from app.database.images import db_create_images_table +from app.database.videos import db_create_videos_table from app.database.yolo_mapping import db_create_YOLO_classes_table # ############################## @@ -52,6 +53,7 @@ def test_db(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]: monkeypatch.setattr("app.config.settings.DATABASE_PATH", db_path) monkeypatch.setattr("app.database.folders.DATABASE_PATH", db_path) monkeypatch.setattr("app.database.images.DATABASE_PATH", db_path) + monkeypatch.setattr("app.database.videos.DATABASE_PATH", db_path) monkeypatch.setattr("app.database.yolo_mapping.DATABASE_PATH", db_path) # Build the real schema rather than a hand-written copy: a divergent @@ -61,6 +63,7 @@ def test_db(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]: db_create_YOLO_classes_table() db_create_folders_table() db_create_images_table() # db_get_all_folder_details LEFT JOINs it + db_create_videos_table() # db_get_all_folder_details LEFT JOINs it too yield db_path finally: @@ -1124,6 +1127,7 @@ def test_db_get_all_folder_details(self, test_db): 0, # taggingCompleted "not_started", # indexing_status (schema default) 2, # image_count + 0, # video_count ) ] From a8b3bc1eb8c78f460ae786f9e2b85fdf0b55613d Mon Sep 17 00:00:00 2001 From: ROHAN PANDEY <95585299+rohan-pandeyy@users.noreply.github.com> Date: Sun, 26 Jul 2026 03:05:00 +0530 Subject: [PATCH 09/10] test: cover video_count and interval bounds; single-source the bounds Address the second-round review feedback: - SearchResults: add the inverse decoupling test (video search fails, photo results stay visible with an inline video error) and assert the empty-query path calls neither search function. - Folder tests: assert the API surfaces video_count (7 for the video-only fixture) and add a DB case with real videos expecting a non-zero count. - Single-source VIDEO_FRAME_INTERVAL_MIN/MAX in settings.py and import them into the preferences schema, so the API bounds can't drift from the sampler's. Add boundary tests (default, min/max, out-of-range rejection, None for partial updates) for both preference models. --- backend/app/config/settings.py | 10 +++- backend/app/schemas/user_preferences.py | 13 ++-- backend/tests/test_folders.py | 33 +++++++++++ backend/tests/test_user_preferences.py | 59 +++++++++++++++++++ .../pages/__tests__/SearchResults.test.tsx | 37 +++++++++++- 5 files changed, 144 insertions(+), 8 deletions(-) diff --git a/backend/app/config/settings.py b/backend/app/config/settings.py index bc3204102..98a1d9083 100644 --- a/backend/app/config/settings.py +++ b/backend/app/config/settings.py @@ -179,8 +179,16 @@ def _get_env_int( # Video keyframe sampling. One frame every N seconds instead of per-frame # inference (30fps = 1800 forward passes per minute of video). The interval is # stretched when a video would exceed the cap, so cost per video is bounded. +# The bounds are the single source of truth for both the sampler default here +# and the user-preferences API schema, so the API can never accept an interval +# the sampler would reject. +VIDEO_FRAME_INTERVAL_MIN = 0.5 +VIDEO_FRAME_INTERVAL_MAX = 300.0 VIDEO_FRAME_INTERVAL_SECONDS = _get_env_float( - "VIDEO_FRAME_INTERVAL_SECONDS", 5.0, min_value=0.5, max_value=300.0 + "VIDEO_FRAME_INTERVAL_SECONDS", + 5.0, + min_value=VIDEO_FRAME_INTERVAL_MIN, + max_value=VIDEO_FRAME_INTERVAL_MAX, ) VIDEO_MAX_FRAMES_PER_VIDEO = _get_env_int( "VIDEO_MAX_FRAMES_PER_VIDEO", 200, min_value=1 diff --git a/backend/app/schemas/user_preferences.py b/backend/app/schemas/user_preferences.py index 73e0fb47b..f9ca558a7 100644 --- a/backend/app/schemas/user_preferences.py +++ b/backend/app/schemas/user_preferences.py @@ -1,12 +1,13 @@ from pydantic import BaseModel, Field from typing import Optional, Literal -from app.config.settings import VIDEO_FRAME_INTERVAL_SECONDS - -# Kept in lockstep with the sampler's config (settings.py) so the API reports, -# validates, and defaults to exactly what video_util_get_frame_interval() uses. -VIDEO_FRAME_INTERVAL_MIN = 0.5 -VIDEO_FRAME_INTERVAL_MAX = 300.0 +# Single-sourced from the sampler's config so the API reports, validates, and +# defaults to exactly what video_util_get_frame_interval() uses. +from app.config.settings import ( + VIDEO_FRAME_INTERVAL_SECONDS, + VIDEO_FRAME_INTERVAL_MIN, + VIDEO_FRAME_INTERVAL_MAX, +) class UserPreferencesData(BaseModel): diff --git a/backend/tests/test_folders.py b/backend/tests/test_folders.py index dc41f1087..951b57681 100644 --- a/backend/tests/test_folders.py +++ b/backend/tests/test_folders.py @@ -651,6 +651,9 @@ def test_get_all_folders_success( assert first_folder["parent_folder_id"] is None assert first_folder["AI_Tagging"] is True assert first_folder["taggingCompleted"] is False + # The video-only fixture folder must surface its media counts. + assert first_folder["image_count"] == 0 + assert first_folder["video_count"] == 7 mock_get_all_folders.assert_called_once() @@ -1131,6 +1134,36 @@ def test_db_get_all_folder_details(self, test_db): ) ] + def test_db_get_all_folder_details_counts_videos(self, test_db): + """A folder with images and videos reports both counts distinctly.""" + conn = sqlite3.connect(test_db) + conn.execute( + """ + INSERT INTO folders (folder_id, folder_path, parent_folder_id, + last_modified_time, AI_Tagging, taggingCompleted) + VALUES (?, ?, ?, ?, ?, ?) + """, + ("folder-id-1", "/home/user/media", None, 1693526400, True, False), + ) + conn.execute( + "INSERT INTO images (id, path, folder_id) VALUES (?, ?, ?)", + ("img-1", "/home/user/media/a.jpg", "folder-id-1"), + ) + conn.executemany( + "INSERT INTO videos (id, path, folder_id) VALUES (?, ?, ?)", + [ + ("vid-1", "/home/user/media/a.mp4", "folder-id-1"), + ("vid-2", "/home/user/media/b.mp4", "folder-id-1"), + ], + ) + conn.commit() + conn.close() + + (row,) = db_get_all_folder_details() + image_count, video_count = row[7], row[8] + assert image_count == 1 + assert video_count == 2 + def test_db_get_direct_child_folders(self, test_db): conn = sqlite3.connect(test_db) conn.execute( diff --git a/backend/tests/test_user_preferences.py b/backend/tests/test_user_preferences.py index e77ec15f4..911a956a4 100644 --- a/backend/tests/test_user_preferences.py +++ b/backend/tests/test_user_preferences.py @@ -504,3 +504,62 @@ def test_unsupported_http_methods(self, method, endpoint): """Test that unsupported HTTP methods return 405.""" response = client.request(method, endpoint) assert response.status_code == 405 + + +class TestVideoFrameIntervalValidation: + """Boundary coverage for the video keyframe interval on both preference + models, guarding the sampling API contract.""" + + def test_default_matches_sampler_config(self): + from app.schemas.user_preferences import UserPreferencesData + from app.config.settings import VIDEO_FRAME_INTERVAL_SECONDS + + assert ( + UserPreferencesData().Video_Frame_Interval == VIDEO_FRAME_INTERVAL_SECONDS + ) + + @pytest.mark.parametrize( + "value", + [ + 0.5, # exact minimum + 300.0, # exact maximum + 5.0, + ], + ) + def test_accepts_in_range_values(self, value): + from app.schemas.user_preferences import ( + UserPreferencesData, + UpdateUserPreferencesRequest, + ) + + assert UserPreferencesData(Video_Frame_Interval=value).Video_Frame_Interval == ( + value + ) + assert ( + UpdateUserPreferencesRequest( + Video_Frame_Interval=value + ).Video_Frame_Interval + == value + ) + + @pytest.mark.parametrize("value", [0.4, 300.1, 0.0, -1.0]) + def test_rejects_out_of_range_values(self, value): + from pydantic import ValidationError + from app.schemas.user_preferences import ( + UserPreferencesData, + UpdateUserPreferencesRequest, + ) + + with pytest.raises(ValidationError): + UserPreferencesData(Video_Frame_Interval=value) + with pytest.raises(ValidationError): + UpdateUserPreferencesRequest(Video_Frame_Interval=value) + + def test_partial_update_allows_none(self): + from app.schemas.user_preferences import UpdateUserPreferencesRequest + + assert UpdateUserPreferencesRequest().Video_Frame_Interval is None + assert ( + UpdateUserPreferencesRequest(Video_Frame_Interval=None).Video_Frame_Interval + is None + ) diff --git a/frontend/src/pages/__tests__/SearchResults.test.tsx b/frontend/src/pages/__tests__/SearchResults.test.tsx index 2e8838ed2..043ce7e02 100644 --- a/frontend/src/pages/__tests__/SearchResults.test.tsx +++ b/frontend/src/pages/__tests__/SearchResults.test.tsx @@ -38,8 +38,9 @@ describe('SearchResults Page', () => { test('renders empty state when no query is provided', async () => { renderWithQuery(''); - // Should not call the API if query is empty + // Neither search runs when there's no query expect(searchImagesByTag).not.toHaveBeenCalled(); + expect(searchVideosByTag).not.toHaveBeenCalled(); // Check for empty state message expect( @@ -165,6 +166,40 @@ describe('SearchResults Page', () => { ); }); + test('a video-search error is shown inline and does not hide photo results', async () => { + (searchImagesByTag as jest.Mock).mockResolvedValue({ + success: true, + data: [ + { + id: '1', + path: '/img1.jpg', + thumbnailPath: '/thumb1.jpg', + tags: ['cat'], + }, + ], + }); + (searchVideosByTag as jest.Mock).mockRejectedValue( + new Error('Network Error'), + ); + + renderWithQuery('cat'); + + await waitFor( + () => { + // Inline video error, not the full-screen takeover... + expect( + screen.getByText(/Couldn't load video results/i), + ).toBeInTheDocument(); + expect(screen.queryByText(/^Search Failed$/i)).not.toBeInTheDocument(); + // ...and the photos still render. + expect( + screen.queryByText(/Couldn't load photo results/i), + ).not.toBeInTheDocument(); + }, + { timeout: 3000 }, + ); + }); + test('the Search Failed takeover appears only when both searches fail', async () => { (searchImagesByTag as jest.Mock).mockRejectedValue( new Error('Network Error'), From 6e0b5078aec0954c6ed881bc6706ab4d46c19d0d Mon Sep 17 00:00:00 2001 From: ROHAN PANDEY <95585299+rohan-pandeyy@users.noreply.github.com> Date: Sun, 26 Jul 2026 03:10:41 +0530 Subject: [PATCH 10/10] test: exercise both sides of the folder media join Strengthen the video-count test per review: the mixed folder now holds 2 images and 2 videos (the join fans out to 4 rows, so a non-DISTINCT count would over-report), and a separate video-only folder pins image_count=0. --- backend/tests/test_folders.py | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/backend/tests/test_folders.py b/backend/tests/test_folders.py index 951b57681..18585de83 100644 --- a/backend/tests/test_folders.py +++ b/backend/tests/test_folders.py @@ -1135,34 +1135,46 @@ def test_db_get_all_folder_details(self, test_db): ] def test_db_get_all_folder_details_counts_videos(self, test_db): - """A folder with images and videos reports both counts distinctly.""" + """Distinct image/video counts on both sides of the media join. + + The mixed folder deliberately has >1 of each: the images x videos + join fans out to 2*2=4 rows, so counting anything but DISTINCT ids + would over-report. A separate video-only folder pins image_count=0. + """ conn = sqlite3.connect(test_db) - conn.execute( + conn.executemany( """ INSERT INTO folders (folder_id, folder_path, parent_folder_id, last_modified_time, AI_Tagging, taggingCompleted) VALUES (?, ?, ?, ?, ?, ?) """, - ("folder-id-1", "/home/user/media", None, 1693526400, True, False), + [ + ("mixed", "/home/user/media", None, 1693526400, True, False), + ("videos-only", "/home/user/clips", None, 1693526400, True, False), + ], ) - conn.execute( + conn.executemany( "INSERT INTO images (id, path, folder_id) VALUES (?, ?, ?)", - ("img-1", "/home/user/media/a.jpg", "folder-id-1"), + [ + ("img-1", "/home/user/media/a.jpg", "mixed"), + ("img-2", "/home/user/media/b.jpg", "mixed"), + ], ) conn.executemany( "INSERT INTO videos (id, path, folder_id) VALUES (?, ?, ?)", [ - ("vid-1", "/home/user/media/a.mp4", "folder-id-1"), - ("vid-2", "/home/user/media/b.mp4", "folder-id-1"), + ("vid-1", "/home/user/media/a.mp4", "mixed"), + ("vid-2", "/home/user/media/b.mp4", "mixed"), + ("vid-3", "/home/user/clips/c.mp4", "videos-only"), ], ) conn.commit() conn.close() - (row,) = db_get_all_folder_details() - image_count, video_count = row[7], row[8] - assert image_count == 1 - assert video_count == 2 + # (image_count, video_count) keyed by folder_id, order-independent. + counts = {row[0]: (row[7], row[8]) for row in db_get_all_folder_details()} + assert counts["mixed"] == (2, 2) + assert counts["videos-only"] == (0, 1) def test_db_get_direct_child_folders(self, test_db): conn = sqlite3.connect(test_db)