diff --git a/backend/app/config/settings.py b/backend/app/config/settings.py index bcb80cd3a..98a1d9083 100644 --- a/backend/app/config/settings.py +++ b/backend/app/config/settings.py @@ -39,6 +39,7 @@ else: DATABASE_PATH = os.path.join(user_data_dir("PictoPy"), "database", "PictoPy.db") THUMBNAIL_IMAGES_PATH = os.path.join(user_data_dir("PictoPy"), "thumbnails") +VIDEO_FRAMES_PATH = os.path.join(user_data_dir("PictoPy"), "video_frames") IMAGES_PATH = "./images" @@ -175,6 +176,33 @@ def _get_env_int( } SEMANTIC_DEFAULT_THRESHOLD = 5e-05 +# Video keyframe sampling. One frame every N seconds instead of per-frame +# inference (30fps = 1800 forward passes per minute of video). The interval is +# stretched when a video would exceed the cap, so cost per video is bounded. +# The bounds are the single source of truth for both the sampler default here +# and the user-preferences API schema, so the API can never accept an interval +# the sampler would reject. +VIDEO_FRAME_INTERVAL_MIN = 0.5 +VIDEO_FRAME_INTERVAL_MAX = 300.0 +VIDEO_FRAME_INTERVAL_SECONDS = _get_env_float( + "VIDEO_FRAME_INTERVAL_SECONDS", + 5.0, + min_value=VIDEO_FRAME_INTERVAL_MIN, + max_value=VIDEO_FRAME_INTERVAL_MAX, +) +VIDEO_MAX_FRAMES_PER_VIDEO = _get_env_int( + "VIDEO_MAX_FRAMES_PER_VIDEO", 200, min_value=1 +) +# Frames are saved above SigLIP2's largest input (384) so a checkpoint swap +# doesn't require re-extraction. +VIDEO_FRAME_MAX_DIMENSION = _get_env_int("VIDEO_FRAME_MAX_DIMENSION", 640, min_value=64) +# A tag must appear in this many frames to describe the video; drops one-off +# detections from a single unlucky keyframe. +VIDEO_TAG_MIN_FRAME_SUPPORT = _get_env_int( + "VIDEO_TAG_MIN_FRAME_SUPPORT", 2, min_value=1 +) +VIDEO_TAG_TOP_K = _get_env_int("VIDEO_TAG_TOP_K", 15, min_value=1) + # Clustering Configuration PICTO_CLUSTERING_EPS = _get_env_float("PICTO_CLUSTERING_EPS", 0.75, min_value=0.0) PICTO_CLUSTERING_MIN_SAMPLES = _get_env_int( diff --git a/backend/app/database/folders.py b/backend/app/database/folders.py index 70344f0e4..68118e796 100644 --- a/backend/app/database/folders.py +++ b/backend/app/database/folders.py @@ -408,30 +408,35 @@ def db_get_folder_ids_by_paths( def db_get_all_folder_details() -> ( - List[Tuple[str, str, Optional[str], int, bool, Optional[bool], str, int]] + List[Tuple[str, str, Optional[str], int, bool, Optional[bool], str, int, int]] ): """ Get all folder details including folder_id, folder_path, parent_folder_id, - last_modified_time, AI_Tagging, taggingCompleted, and image_count. + last_modified_time, AI_Tagging, taggingCompleted, indexing_status, + image_count and video_count. Returns list of tuples with all folder information. """ conn = sqlite3.connect(DATABASE_PATH) cursor = conn.cursor() try: + # COUNT(DISTINCT ...) because joining both media tables multiplies the + # rows: a folder with 3 images and 4 videos yields 12 join rows. cursor.execute( """ - SELECT - f.folder_id, - f.folder_path, - f.parent_folder_id, - f.last_modified_time, - f.AI_Tagging, + SELECT + f.folder_id, + f.folder_path, + f.parent_folder_id, + f.last_modified_time, + f.AI_Tagging, f.taggingCompleted, f.indexing_status, - COUNT(i.id) as image_count + COUNT(DISTINCT i.id) as image_count, + COUNT(DISTINCT v.id) as video_count FROM folders f LEFT JOIN images i ON f.folder_id = i.folder_id + LEFT JOIN videos v ON f.folder_id = v.folder_id GROUP BY f.folder_id ORDER BY f.folder_path """ diff --git a/backend/app/database/video_frames.py b/backend/app/database/video_frames.py new file mode 100644 index 000000000..eee36ed96 --- /dev/null +++ b/backend/app/database/video_frames.py @@ -0,0 +1,535 @@ +"""Sampled video keyframes and the tags derived from them. + +Frames are deliberately kept out of the images table: they would otherwise +leak into the photo gallery, memories and face clustering. These tables +mirror their image counterparts so the existing tagging passes port over +with the same shapes. +""" + +import sqlite3 +from typing import Dict, List, Optional, Tuple + +import numpy as np + +from app.database.images import _connect +from app.database.semantic_labels import SEMANTIC_CLASS_ID_OFFSET +from app.logging.setup_logging import get_logger + +logger = get_logger(__name__) + + +def db_create_video_frames_tables() -> None: + conn = None + try: + conn = _connect() + cursor = conn.cursor() + + # frame_path is nullable so the purge action can drop the JPEGs + # without destroying the embeddings that depend on them. + cursor.execute( + """ + CREATE TABLE IF NOT EXISTS video_frames ( + id TEXT PRIMARY KEY, + video_id TEXT NOT NULL, + frame_path TEXT UNIQUE, + timestamp_sec REAL, + frame_index INTEGER, + isEmbedded BOOLEAN DEFAULT 0, + FOREIGN KEY (video_id) REFERENCES videos(id) ON DELETE CASCADE + ) + """ + ) + cursor.execute( + "CREATE INDEX IF NOT EXISTS ix_video_frames_video_id " + "ON video_frames(video_id)" + ) + + # Same raw-float32 blob format as image_embeddings.embedding. + cursor.execute( + """ + CREATE TABLE IF NOT EXISTS video_frame_embeddings ( + frame_id TEXT PRIMARY KEY, + model_version TEXT NOT NULL, + embedding BLOB NOT NULL, + scored_signature TEXT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (frame_id) REFERENCES video_frames(id) ON DELETE CASCADE + ) + """ + ) + cursor.execute( + "CREATE INDEX IF NOT EXISTS ix_video_frame_embeddings_model_version " + "ON video_frame_embeddings(model_version)" + ) + + # Video-level tags, aggregated from the frames at write time so tag + # queries stay a plain join. score is NULL for YOLO rows and the best + # frame's match score for semantic rows, matching image_classes. + cursor.execute( + """ + CREATE TABLE IF NOT EXISTS video_classes ( + video_id TEXT, + class_id INTEGER, + score REAL, + frame_count INTEGER, + PRIMARY KEY (video_id, class_id), + FOREIGN KEY (video_id) REFERENCES videos(id) ON DELETE CASCADE, + FOREIGN KEY (class_id) REFERENCES mappings(class_id) ON DELETE CASCADE + ) + """ + ) + + # Display cut mirroring image_classes_display: all YOLO tags, but only + # the top-SEMANTIC_DISPLAY_TOP_K semantic tags per video. Recreated at + # startup so setting changes apply without re-scoring. + from app.config.settings import SEMANTIC_DISPLAY_TOP_K + + cursor.execute("DROP VIEW IF EXISTS video_classes_display") + cursor.execute( + f""" + CREATE VIEW video_classes_display AS + SELECT video_id, class_id FROM ( + SELECT video_id, class_id, + ROW_NUMBER() OVER ( + PARTITION BY video_id ORDER BY score DESC + ) AS display_rank + FROM video_classes WHERE score IS NOT NULL + ) WHERE display_rank <= {int(SEMANTIC_DISPLAY_TOP_K)} + UNION ALL + SELECT video_id, class_id FROM video_classes WHERE score IS NULL + """ + ) + + conn.commit() + finally: + if conn: + conn.close() + + +def db_get_untagged_videos() -> List[Dict[str, str]]: + """Videos in AI-tagging-enabled folders that have not been tagged yet.""" + conn = None + try: + conn = _connect() + cursor = conn.cursor() + cursor.execute( + """ + SELECT v.id, v.path + FROM videos v + JOIN folders f ON v.folder_id = f.folder_id + WHERE f.AI_Tagging = TRUE + AND v.isTagged = FALSE + """ + ) + return [{"id": video_id, "path": path} for video_id, path in cursor.fetchall()] + finally: + if conn: + conn.close() + + +def db_mark_videos_tagged(video_ids: List[str]) -> bool: + if not video_ids: + return True + + conn = None + try: + conn = _connect() + cursor = conn.cursor() + placeholders = ",".join("?" for _ in video_ids) + cursor.execute( + f"UPDATE videos SET isTagged = 1 WHERE id IN ({placeholders})", + video_ids, + ) + conn.commit() + return True + except sqlite3.Error as e: + logger.error(f"Error marking videos as tagged: {e}") + if conn: + conn.rollback() + return False + finally: + if conn: + conn.close() + + +def db_bulk_insert_video_frames(frame_records: List[dict]) -> bool: + """Insert sampled frame rows: id, video_id, frame_path, timestamp_sec, + frame_index.""" + if not frame_records: + return True + + conn = None + try: + conn = _connect() + cursor = conn.cursor() + cursor.executemany( + """ + INSERT INTO video_frames + (id, video_id, frame_path, timestamp_sec, frame_index) + VALUES (:id, :video_id, :frame_path, :timestamp_sec, :frame_index) + ON CONFLICT(frame_path) DO UPDATE SET + video_id=excluded.video_id, + timestamp_sec=excluded.timestamp_sec, + frame_index=excluded.frame_index + """, + frame_records, + ) + conn.commit() + return True + except sqlite3.Error as e: + logger.error(f"Error inserting video frames: {e}") + if conn: + conn.rollback() + return False + finally: + if conn: + conn.close() + + +def db_delete_frames_for_videos(video_ids: List[str]) -> bool: + """Drop a video's frame rows (and, by cascade, their embeddings) so a + re-tag starts from a clean sample.""" + if not video_ids: + return True + + conn = None + try: + conn = _connect() + cursor = conn.cursor() + placeholders = ",".join("?" for _ in video_ids) + cursor.execute( + f"DELETE FROM video_frames WHERE video_id IN ({placeholders})", + video_ids, + ) + conn.commit() + return True + except sqlite3.Error as e: + logger.error(f"Error deleting video frames: {e}") + if conn: + conn.rollback() + return False + finally: + if conn: + conn.close() + + +def db_get_unembedded_video_frames() -> List[Dict[str, str]]: + """Frames still needing a SigLIP2 embedding. Purged frames (frame_path + NULL) are excluded -- there is nothing left on disk to embed.""" + conn = None + try: + conn = _connect() + cursor = conn.cursor() + cursor.execute( + """ + SELECT vf.id, vf.frame_path + FROM video_frames vf + JOIN videos v ON vf.video_id = v.id + JOIN folders f ON v.folder_id = f.folder_id + WHERE f.AI_Tagging = TRUE + AND vf.isEmbedded = FALSE + AND vf.frame_path IS NOT NULL + """ + ) + return [ + {"id": frame_id, "frame_path": frame_path} + for frame_id, frame_path in cursor.fetchall() + ] + finally: + if conn: + conn.close() + + +def db_mark_video_frames_embedded(frame_ids: List[str]) -> bool: + if not frame_ids: + return True + + conn = None + try: + conn = _connect() + cursor = conn.cursor() + # Chunked by 500 to stay under SQLite's 999-variable limit, matching + # db_mark_images_embedded. + chunk_size = 500 + for i in range(0, len(frame_ids), chunk_size): + chunk = frame_ids[i : i + chunk_size] + placeholders = ",".join("?" for _ in chunk) + cursor.execute( + f"UPDATE video_frames SET isEmbedded = 1 WHERE id IN ({placeholders})", + chunk, + ) + conn.commit() + return True + except sqlite3.Error as e: + logger.error(f"Error marking video frames as embedded: {e}") + if conn: + conn.rollback() + return False + finally: + if conn: + conn.close() + + +def db_upsert_video_frame_embeddings( + rows: List[Tuple[str, str, np.ndarray]], +) -> None: + conn = None + try: + conn = _connect() + cursor = conn.cursor() + cursor.executemany( + """ + INSERT INTO video_frame_embeddings (frame_id, model_version, embedding) + VALUES (?, ?, ?) + ON CONFLICT(frame_id) DO UPDATE SET + model_version = excluded.model_version, + embedding = excluded.embedding, + scored_signature = NULL, + created_at = CURRENT_TIMESTAMP + """, + [ + ( + frame_id, + model_version, + np.ascontiguousarray(embedding, dtype=np.float32).tobytes(), + ) + for frame_id, model_version, embedding in rows + ], + ) + conn.commit() + finally: + if conn: + conn.close() + + +def db_write_video_classes(video_id: str, pairs: List[Tuple[int, int]]) -> None: + """Replace a video's YOLO tag rows with (class_id, frame_count) pairs. + Semantic rows (class_id at or above the offset) are never touched.""" + conn = None + try: + conn = _connect() + cursor = conn.cursor() + cursor.execute( + "DELETE FROM video_classes WHERE video_id = ? AND class_id < ?", + (video_id, SEMANTIC_CLASS_ID_OFFSET), + ) + cursor.executemany( + "INSERT OR REPLACE INTO video_classes " + "(video_id, class_id, score, frame_count) VALUES (?, ?, NULL, ?)", + [(video_id, class_id, frame_count) for class_id, frame_count in pairs], + ) + conn.commit() + finally: + if conn: + conn.close() + + +def db_write_video_semantic_scores( + video_id: str, pairs: List[Tuple[int, float, int]], signature: str +) -> None: + """Replace a video's semantic tag rows with (class_id, score, frame_count) + triples and stamp the signature on every frame of that video. YOLO rows + are never touched -- mirrors db_write_image_semantic_scores.""" + conn = None + try: + conn = _connect() + cursor = conn.cursor() + cursor.execute( + "DELETE FROM video_classes WHERE video_id = ? AND class_id >= ?", + (video_id, SEMANTIC_CLASS_ID_OFFSET), + ) + cursor.executemany( + "INSERT OR REPLACE INTO video_classes " + "(video_id, class_id, score, frame_count) VALUES (?, ?, ?, ?)", + [ + (video_id, class_id, score, frame_count) + for class_id, score, frame_count in pairs + ], + ) + cursor.execute( + """ + UPDATE video_frame_embeddings SET scored_signature = ? + WHERE frame_id IN (SELECT id FROM video_frames WHERE video_id = ?) + """, + (signature, video_id), + ) + conn.commit() + finally: + if conn: + conn.close() + + +def db_get_videos_needing_scoring( + model_version: str, signature: str, limit: int +) -> List[str]: + """Videos with at least one frame embedding whose semantic scores are + missing or computed against another vocabulary/label state.""" + conn = None + try: + conn = _connect() + cursor = conn.cursor() + cursor.execute( + """ + SELECT DISTINCT vf.video_id + FROM video_frames vf + JOIN video_frame_embeddings e ON vf.id = e.frame_id + WHERE e.model_version = ? + AND IFNULL(e.scored_signature, '') != ? + LIMIT ? + """, + (model_version, signature, limit), + ) + return [row[0] for row in cursor.fetchall()] + finally: + if conn: + conn.close() + + +def db_get_frame_embeddings_for_video(video_id: str, model_version: str) -> np.ndarray: + """All of one video's frame embeddings as an [N, D] matrix.""" + conn = None + try: + conn = _connect() + cursor = conn.cursor() + cursor.execute( + """ + SELECT e.embedding + FROM video_frames vf + JOIN video_frame_embeddings e ON vf.id = e.frame_id + WHERE vf.video_id = ? AND e.model_version = ? + ORDER BY vf.frame_index + """, + (video_id, model_version), + ) + rows = cursor.fetchall() + if not rows: + return np.empty((0, 0), dtype=np.float32) + return np.vstack([np.frombuffer(blob, dtype=np.float32) for (blob,) in rows]) + finally: + if conn: + conn.close() + + +def db_get_all_frame_embeddings( + model_version: str, +) -> Tuple[List[str], List[float], np.ndarray]: + """Every frame embedding with its video and timestamp, for semantic + search. Returns (video_ids, timestamps, matrix) aligned by row.""" + conn = None + try: + conn = _connect() + cursor = conn.cursor() + cursor.execute( + """ + SELECT vf.video_id, vf.timestamp_sec, e.embedding + FROM video_frames vf + JOIN video_frame_embeddings e ON vf.id = e.frame_id + WHERE e.model_version = ? + """, + (model_version,), + ) + rows = cursor.fetchall() + if not rows: + return [], [], np.empty((0, 0), dtype=np.float32) + + video_ids = [row[0] for row in rows] + timestamps = [row[1] for row in rows] + matrix = np.vstack([np.frombuffer(row[2], dtype=np.float32) for row in rows]) + return video_ids, timestamps, matrix + finally: + if conn: + conn.close() + + +def db_get_video_tags(video_ids: Optional[List[str]] = None) -> Dict[str, List[str]]: + """Display tag names per video. Pass video_ids to restrict, or None for + every video.""" + base_query = """ + SELECT vc.video_id, m.name + FROM video_classes_display vc + JOIN mappings m ON vc.class_id = m.class_id + """ + order_by = " ORDER BY vc.video_id, m.name" + + conn = None + try: + conn = _connect() + cursor = conn.cursor() + + tags: Dict[str, List[str]] = {} + + def collect(query: str, params: List[str]) -> None: + cursor.execute(query, params) + for video_id, name in cursor.fetchall(): + tags.setdefault(video_id, []).append(name) + + if video_ids is None: + collect(base_query + order_by, []) + else: + if not video_ids: + return {} + # Chunk below SQLite's 999-placeholder limit so libraries with + # more than ~999 videos don't error out and lose every tag. + chunk_size = 500 + for i in range(0, len(video_ids), chunk_size): + chunk = video_ids[i : i + chunk_size] + placeholders = ",".join("?" for _ in chunk) + collect( + base_query + f" WHERE vc.video_id IN ({placeholders})" + order_by, + chunk, + ) + + return tags + except sqlite3.Error as e: + logger.error(f"Error getting video tags: {e}") + return {} + finally: + if conn: + conn.close() + + +def db_get_video_ids_by_tag(tag_name: str) -> List[str]: + """Video IDs carrying a tag. Matches against the full stored tag set, not + the display cut, so search isn't limited by SEMANTIC_DISPLAY_TOP_K.""" + conn = None + try: + conn = _connect() + cursor = conn.cursor() + cursor.execute( + """ + SELECT DISTINCT vc.video_id + FROM video_classes vc + JOIN mappings m ON vc.class_id = m.class_id + WHERE LOWER(m.name) = LOWER(?) + """, + (tag_name,), + ) + return [row[0] for row in cursor.fetchall()] + except sqlite3.Error as e: + logger.error(f"Error searching videos by tag: {e}") + raise + finally: + if conn: + conn.close() + + +def db_clear_frame_paths() -> bool: + """Forget where the frame JPEGs were, keeping the rows and their + embeddings so tags and semantic search survive a purge.""" + conn = None + try: + conn = _connect() + cursor = conn.cursor() + cursor.execute( + "UPDATE video_frames SET frame_path = NULL WHERE frame_path IS NOT NULL" + ) + conn.commit() + return True + except sqlite3.Error as e: + logger.error(f"Error clearing video frame paths: {e}") + if conn: + conn.rollback() + return False + finally: + if conn: + conn.close() diff --git a/backend/app/database/videos.py b/backend/app/database/videos.py index e9e528334..f8c875eed 100644 --- a/backend/app/database/videos.py +++ b/backend/app/database/videos.py @@ -44,9 +44,10 @@ def db_create_videos_table() -> None: conn = _connect() cursor = conn.cursor() - # Videos are kept separate from images by design; isTagged is reserved - # for future AI video tagging. thumbnailPath is nullable: videos whose - # codec OpenCV cannot decode are still indexed (frontend shows a placeholder). + # Videos are kept separate from images by design; isTagged tracks the + # keyframe sampling pass (see video_frames). thumbnailPath is nullable: + # videos whose codec OpenCV cannot decode are still indexed (frontend + # shows a placeholder). cursor.execute( """ CREATE TABLE IF NOT EXISTS videos ( @@ -127,34 +128,7 @@ def db_get_all_videos() -> List[dict]: results = cursor.fetchall() - videos = [] - for ( - video_id, - path, - folder_id, - thumbnail_path, - metadata, - is_tagged, - is_favourite, - captured_at, - ) in results: - from app.utils.images import image_util_parse_metadata - - videos.append( - { - "id": video_id, - "path": path, - "folder_id": str(folder_id), - "thumbnailPath": thumbnail_path, - "metadata": image_util_parse_metadata(metadata), - "isTagged": bool(is_tagged), - "isFavourite": bool(is_favourite), - "captured_at": captured_at if captured_at else None, - "tags": None, - } - ) - - return videos + return _build_video_dicts(results) except sqlite3.Error as e: logger.error(f"Error getting all videos: {e}") @@ -163,6 +137,42 @@ def db_get_all_videos() -> List[dict]: conn.close() +def _build_video_dicts(rows: List[Tuple]) -> List[dict]: + """Turn the shared 8-column video SELECT into dicts, attaching each + video's display tags.""" + from app.database.video_frames import db_get_video_tags + from app.utils.images import image_util_parse_metadata + + tags_by_video = db_get_video_tags([row[0] for row in rows]) + + videos = [] + for ( + video_id, + path, + folder_id, + thumbnail_path, + metadata, + is_tagged, + is_favourite, + captured_at, + ) in rows: + videos.append( + { + "id": video_id, + "path": path, + "folder_id": str(folder_id), + "thumbnailPath": thumbnail_path, + "metadata": image_util_parse_metadata(metadata), + "isTagged": bool(is_tagged), + "isFavourite": bool(is_favourite), + "captured_at": captured_at if captured_at else None, + "tags": tags_by_video.get(video_id) or None, + } + ) + + return videos + + def db_get_video_index_by_folder_ids( folder_ids: List[int], ) -> List[Tuple[VideoPath, Optional[str], Optional[str]]]: @@ -200,6 +210,44 @@ def db_get_video_index_by_folder_ids( conn.close() +def db_get_videos_by_ids(video_ids: List[VideoId]) -> List[dict]: + """Get videos by ID, preserving the order of video_ids.""" + if not video_ids: + return [] + + conn = _connect() + cursor = conn.cursor() + + try: + rows = [] + # Chunked by 500 to stay under SQLite's 999-variable limit. + chunk_size = 500 + for i in range(0, len(video_ids), chunk_size): + chunk = video_ids[i : i + chunk_size] + placeholders = ",".join("?" for _ in chunk) + cursor.execute( + f""" + SELECT id, path, folder_id, thumbnailPath, metadata, isTagged, + isFavourite, captured_at + FROM videos + WHERE id IN ({placeholders}) + """, + chunk, + ) + rows.extend(cursor.fetchall()) + + video_lookup = {video["id"]: video for video in _build_video_dicts(rows)} + return [ + video_lookup[video_id] for video_id in video_ids if video_id in video_lookup + ] + + except sqlite3.Error as e: + logger.error(f"Error getting videos by IDs: {e}") + raise + finally: + conn.close() + + def db_get_videos_by_folder_ids( folder_ids: List[int], ) -> List[Tuple[VideoId, VideoPath, str]]: diff --git a/backend/app/routes/folders.py b/backend/app/routes/folders.py index 3780c7eb9..93ab570f1 100644 --- a/backend/app/routes/folders.py +++ b/backend/app/routes/folders.py @@ -44,9 +44,16 @@ image_util_process_untagged_images, image_util_process_unembedded_images, ) -from app.utils.videos import video_util_process_folder_videos +from app.utils.videos import ( + video_util_process_folder_videos, + video_util_process_untagged_videos, + video_util_process_unembedded_frames, +) from app.utils.model_bootstrap import ensure_ai_tagging_models -from app.utils.semantic_labels import semantic_util_score_images +from app.utils.semantic_labels import ( + semantic_util_score_images, + semantic_util_score_videos, +) from app.utils.face_clusters import cluster_util_face_clusters_sync from app.utils.API import API_util_restart_sync_microservice_watcher @@ -107,6 +114,10 @@ def post_AI_tagging_enabled_sequence(): cluster_util_face_clusters_sync() image_util_process_unembedded_images() semantic_util_score_images() + # Videos last: photos are the primary surface, so they finish first. + video_util_process_untagged_videos() + video_util_process_unembedded_frames() + semantic_util_score_videos() except Exception as e: logger.error(f"Error in post processing after AI tagging was enabled: {e}") return False @@ -138,6 +149,9 @@ def post_sync_folder_sequence( cluster_util_face_clusters_sync() image_util_process_unembedded_images() semantic_util_score_images() + video_util_process_untagged_videos() + video_util_process_unembedded_frames() + semantic_util_score_videos() # Restart sync microservice watcher after processing images API_util_restart_sync_microservice_watcher() @@ -471,6 +485,7 @@ def get_all_folders(): tagging_completed, indexing_status, image_count, + video_count, ) = folder_data folders.append( FolderDetails( @@ -482,6 +497,7 @@ def get_all_folders(): taggingCompleted=tagging_completed, indexing_status=indexing_status, image_count=image_count, + video_count=video_count, ) ) diff --git a/backend/app/routes/user_preferences.py b/backend/app/routes/user_preferences.py index 678e8cfc0..982654613 100644 --- a/backend/app/routes/user_preferences.py +++ b/backend/app/routes/user_preferences.py @@ -1,4 +1,5 @@ from fastapi import APIRouter, HTTPException, status +from app.config.settings import VIDEO_FRAME_INTERVAL_SECONDS from app.database.metadata import db_get_metadata, db_update_metadata from app.schemas.user_preferences import ( GetUserPreferencesResponse, @@ -30,6 +31,9 @@ def get_user_preferences(): user_preferences = UserPreferencesData( YOLO_model_size=user_prefs_data.get("YOLO_model_size", "small"), GPU_Acceleration=user_prefs_data.get("GPU_Acceleration", True), + Video_Frame_Interval=user_prefs_data.get( + "Video_Frame_Interval", VIDEO_FRAME_INTERVAL_SECONDS + ), ) return GetUserPreferencesResponse( @@ -58,7 +62,11 @@ def update_user_preferences(request: UpdateUserPreferencesRequest): """Update user preferences in metadata.""" try: # Step 1: Validate that at least one field is provided - if request.YOLO_model_size is None and request.GPU_Acceleration is None: + if ( + request.YOLO_model_size is None + and request.GPU_Acceleration is None + and request.Video_Frame_Interval is None + ): raise ValueError("At least one preference field must be provided") # Step 2: Get current metadata @@ -74,6 +82,9 @@ def update_user_preferences(request: UpdateUserPreferencesRequest): if request.GPU_Acceleration is not None: current_user_prefs["GPU_Acceleration"] = request.GPU_Acceleration + if request.Video_Frame_Interval is not None: + current_user_prefs["Video_Frame_Interval"] = request.Video_Frame_Interval + # Step 5: Update metadata with new user preferences metadata["user_preferences"] = current_user_prefs @@ -94,6 +105,9 @@ def update_user_preferences(request: UpdateUserPreferencesRequest): user_preferences = UserPreferencesData( YOLO_model_size=current_user_prefs.get("YOLO_model_size", "small"), GPU_Acceleration=current_user_prefs.get("GPU_Acceleration", True), + Video_Frame_Interval=current_user_prefs.get( + "Video_Frame_Interval", VIDEO_FRAME_INTERVAL_SECONDS + ), ) return UpdateUserPreferencesResponse( diff --git a/backend/app/routes/videos.py b/backend/app/routes/videos.py index ff7230ed1..750b6dff9 100644 --- a/backend/app/routes/videos.py +++ b/backend/app/routes/videos.py @@ -1,9 +1,10 @@ -from fastapi import APIRouter, HTTPException, status +from fastapi import APIRouter, HTTPException, Query, status from typing import List, Optional from pydantic import BaseModel, ValidationError from app.database.videos import ( db_get_all_videos, + db_get_videos_by_ids, db_toggle_video_favourite_status, db_get_video_by_id, ) @@ -44,6 +45,30 @@ class GetAllVideosResponse(BaseModel): data: List[VideoData] +def _to_video_data(videos: List[dict]) -> List[VideoData]: + """Build per row: one record with unusable metadata shouldn't 500 the + whole listing and hide every other video.""" + video_data = [] + for video in videos: + try: + video_data.append( + VideoData( + id=video["id"], + path=video["path"], + folder_id=video["folder_id"], + thumbnailPath=video["thumbnailPath"], + metadata=video["metadata"], + isFavourite=video.get("isFavourite", False), + tags=video["tags"], + ) + ) + except ValidationError as e: + logger.warning( + f"Skipping video {video.get('id')} with invalid metadata: {e}" + ) + return video_data + + @router.get( "/", response_model=GetAllVideosResponse, @@ -52,28 +77,7 @@ class GetAllVideosResponse(BaseModel): def get_all_videos(): """Get all videos from the database.""" try: - videos = db_get_all_videos() - - # Build per row: one record with unusable metadata shouldn't 500 the - # whole listing and hide every other video. - video_data = [] - for video in videos: - try: - video_data.append( - VideoData( - id=video["id"], - path=video["path"], - folder_id=video["folder_id"], - thumbnailPath=video["thumbnailPath"], - metadata=video["metadata"], - isFavourite=video.get("isFavourite", False), - tags=video["tags"], - ) - ) - except ValidationError as e: - logger.warning( - f"Skipping video {video.get('id')} with invalid metadata: {e}" - ) + video_data = _to_video_data(db_get_all_videos()) return GetAllVideosResponse( success=True, @@ -92,6 +96,213 @@ def get_all_videos(): ) +@router.get( + "/search", + response_model=GetAllVideosResponse, + responses={code: {"model": ErrorResponse} for code in [400, 500]}, +) +def search_videos_by_tag(tag: str = Query(..., description="Tag name to search for")): + """Search videos by tag name.""" + try: + from app.database.video_frames import db_get_video_ids_by_tag + + video_data = _to_video_data(db_get_videos_by_ids(db_get_video_ids_by_tag(tag))) + + return GetAllVideosResponse( + success=True, + message=f"Successfully retrieved {len(video_data)} videos for tag '{tag}'", + data=video_data, + ) + + except Exception as e: + logger.error(f"Error searching videos: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=ErrorResponse( + success=False, + error="Internal server error", + message="Unable to search videos due to an internal error", + ).model_dump(), + ) + + +class ScoredVideoData(VideoData): + score: float + # When in the video the best match was found -- what a future "jump to + # this moment" deep link needs. + best_frame_timestamp: Optional[float] = None + + +class SemanticSearchData(BaseModel): + videos: List[ScoredVideoData] + total: int + + +class SemanticSearchResponse(BaseModel): + success: bool + message: str + data: SemanticSearchData + + +@router.get( + "/semantic-search", + response_model=SemanticSearchResponse, + responses={code: {"model": ErrorResponse} for code in [400, 404, 500]}, +) +def semantic_search_videos( + query: str = Query(..., min_length=1, description="Query text to search for") +): + """Semantic search videos by query text using SigLIP2 keyframe embeddings.""" + try: + import os + import numpy as np + + from app.config.settings import ( + SIGLIP2_ACTIVE_CHECKPOINT, + SIGLIP2_SCORING_METADATA, + SIGLIP2_MATCH_THRESHOLD, + SIGLIP2_QUERY_TEMPLATE, + ) + from app.database.video_frames import db_get_all_frame_embeddings + from app.models.model_registry import ( + get_siglip2_registry_keys, + get_siglip2_tokenizer_key, + get_model_path, + ) + from app.utils.SigLIP import ( + siglip_util_tokenize_query, + siglip_util_get_text_model, + ) + + _, text_key = get_siglip2_registry_keys(SIGLIP2_ACTIVE_CHECKPOINT) + tokenizer_key = get_siglip2_tokenizer_key(SIGLIP2_ACTIVE_CHECKPOINT) + text_model_path = get_model_path(text_key) + tokenizer_model_path = get_model_path(tokenizer_key) + + for path, missing in ( + (text_model_path, "SigLIP2 text model"), + (tokenizer_model_path, "SigLIP2 tokenizer"), + ): + if not os.path.exists(path): + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=ErrorResponse( + success=False, + error="Not Found", + message=f"semantic search unavailable: {missing} not installed", + ).model_dump(), + ) + + metadata = SIGLIP2_SCORING_METADATA[SIGLIP2_ACTIVE_CHECKPOINT] + video_ids, timestamps, frame_matrix = db_get_all_frame_embeddings( + metadata["model_version"] + ) + + if not video_ids: + return SemanticSearchResponse( + success=True, + message="No video frames have been embedded yet.", + data=SemanticSearchData(videos=[], total=0), + ) + + text_model = siglip_util_get_text_model(text_model_path, text_key) + input_ids, attention_mask = siglip_util_tokenize_query( + SIGLIP2_QUERY_TEMPLATE.format(query=query) + ) + query_vector = np.asarray( + text_model.get_embedding(input_ids, attention_mask), dtype=np.float32 + ).flatten() + + logits = ( + frame_matrix @ query_vector * np.exp(metadata["logit_scale"]) + + metadata["logit_bias"] + ) + scores = 1.0 / (1.0 + np.exp(-logits)) + + # Max-pool per video: a video matches as well as its best frame does. + best: dict = {} + for i, video_id in enumerate(video_ids): + score = float(scores[i]) + if score < SIGLIP2_MATCH_THRESHOLD: + continue + if video_id not in best or score > best[video_id][0]: + best[video_id] = (score, timestamps[i]) + + ranked = sorted(best.items(), key=lambda item: item[1][0], reverse=True) + # Keyed rather than zipped: db_get_videos_by_ids drops IDs it can't + # find, which would silently shift every score onto the wrong video. + by_id = { + video.id: video + for video in _to_video_data( + db_get_videos_by_ids([video_id for video_id, _ in ranked]) + ) + } + + scored = [ + ScoredVideoData( + **by_id[video_id].model_dump(), + score=score, + best_frame_timestamp=timestamp, + ) + for video_id, (score, timestamp) in ranked + if video_id in by_id + ] + + return SemanticSearchResponse( + success=True, + message=f"Found {len(scored)} matching videos", + data=SemanticSearchData(videos=scored, total=len(scored)), + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error in video semantic search: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=ErrorResponse( + success=False, + error="Internal server error", + message=f"Unable to search videos: {str(e)}", + ).model_dump(), + ) + + +class PurgeFrameCacheResponse(BaseModel): + success: bool + message: str + bytes_reclaimed: int + + +@router.post( + "/purge-frame-cache", + response_model=PurgeFrameCacheResponse, + responses={500: {"model": ErrorResponse}}, +) +def purge_frame_cache(): + """Delete the sampled keyframe JPEGs. Tags and semantic search survive: + only the on-disk frames are removed, not their embeddings.""" + try: + from app.utils.videos import video_util_purge_frame_cache + + reclaimed = video_util_purge_frame_cache() + return PurgeFrameCacheResponse( + success=True, + message=f"Reclaimed {reclaimed} bytes of video frame cache", + bytes_reclaimed=reclaimed, + ) + except Exception as e: + logger.error(f"Error purging video frame cache: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=ErrorResponse( + success=False, + error="Internal server error", + message=f"Unable to purge video frame cache: {str(e)}", + ).model_dump(), + ) + + class ToggleFavouriteRequest(BaseModel): video_id: str diff --git a/backend/app/schemas/folders.py b/backend/app/schemas/folders.py index b64c57ea1..07fac02ac 100644 --- a/backend/app/schemas/folders.py +++ b/backend/app/schemas/folders.py @@ -31,6 +31,7 @@ class FolderDetails(BaseModel): AI_Tagging: bool taggingCompleted: Optional[bool] = None image_count: int = 0 + video_count: int = 0 indexing_status: Optional[str] = ( "completed" # Default to "completed" if not provided ) diff --git a/backend/app/schemas/user_preferences.py b/backend/app/schemas/user_preferences.py index d24191d78..f9ca558a7 100644 --- a/backend/app/schemas/user_preferences.py +++ b/backend/app/schemas/user_preferences.py @@ -1,12 +1,27 @@ -from pydantic import BaseModel +from pydantic import BaseModel, Field from typing import Optional, Literal +# Single-sourced from the sampler's config so the API reports, validates, and +# defaults to exactly what video_util_get_frame_interval() uses. +from app.config.settings import ( + VIDEO_FRAME_INTERVAL_SECONDS, + VIDEO_FRAME_INTERVAL_MIN, + VIDEO_FRAME_INTERVAL_MAX, +) + class UserPreferencesData(BaseModel): """User preferences data structure""" YOLO_model_size: Literal["nano", "small", "medium"] = "small" GPU_Acceleration: bool = True + # Seconds between sampled video keyframes. Lower means finer tag coverage + # and proportionally more inference per video. + Video_Frame_Interval: float = Field( + default=VIDEO_FRAME_INTERVAL_SECONDS, + ge=VIDEO_FRAME_INTERVAL_MIN, + le=VIDEO_FRAME_INTERVAL_MAX, + ) class GetUserPreferencesResponse(BaseModel): @@ -22,6 +37,9 @@ class UpdateUserPreferencesRequest(BaseModel): YOLO_model_size: Optional[Literal["nano", "small", "medium"]] = None GPU_Acceleration: Optional[bool] = None + Video_Frame_Interval: Optional[float] = Field( + default=None, ge=VIDEO_FRAME_INTERVAL_MIN, le=VIDEO_FRAME_INTERVAL_MAX + ) class UpdateUserPreferencesResponse(BaseModel): diff --git a/backend/app/utils/semantic_labels.py b/backend/app/utils/semantic_labels.py index 2963fcfff..d7f4a174d 100644 --- a/backend/app/utils/semantic_labels.py +++ b/backend/app/utils/semantic_labels.py @@ -128,6 +128,57 @@ def semantic_util_build_label_embeddings() -> None: logger.error(f"Error building semantic label embeddings: {e}") +def _load_label_scoring_context(): + """Everything a scoring pass needs: the cached label matrix, per-label + thresholds, the signature that invalidates stale scores, and the + checkpoint's sigmoid calibration. Returns None when no label embeddings + are cached for the active checkpoint. + """ + import numpy as np + from app.config.settings import ( + SIGLIP2_ACTIVE_CHECKPOINT, + SIGLIP2_SCORING_METADATA, + SEMANTIC_SCORE_TOP_K, + SEMANTIC_BUCKET_THRESHOLDS, + SEMANTIC_DEFAULT_THRESHOLD, + ) + from app.database.semantic_labels import db_get_active_label_embeddings + + metadata = SIGLIP2_SCORING_METADATA[SIGLIP2_ACTIVE_CHECKPOINT] + model_version = metadata["model_version"] + + meta, label_matrix = db_get_active_label_embeddings(model_version) + if not meta: + return None + + thresholds = np.array( + [ + ( + override + if override is not None + else SEMANTIC_BUCKET_THRESHOLDS.get( + category, SEMANTIC_DEFAULT_THRESHOLD + ) + ) + for _, category, override in meta + ], + dtype=np.float32, + ) + signature = _scoring_signature( + model_version, SEMANTIC_SCORE_TOP_K, thresholds, meta, label_matrix + ) + + return { + "model_version": model_version, + "meta": meta, + "label_matrix": label_matrix, + "thresholds": thresholds, + "signature": signature, + "logit_scale": np.exp(metadata["logit_scale"]), + "logit_bias": metadata["logit_bias"], + } + + def semantic_util_score_images() -> None: """Score embedded images against the cached label matrix and write top-K above-threshold tags as image_classes rows. @@ -139,49 +190,26 @@ def semantic_util_score_images() -> None: """ import time import numpy as np - from app.config.settings import ( - SIGLIP2_ACTIVE_CHECKPOINT, - SIGLIP2_SCORING_METADATA, - SEMANTIC_SCORE_TOP_K, - SEMANTIC_BUCKET_THRESHOLDS, - SEMANTIC_DEFAULT_THRESHOLD, - ) - from app.database.semantic_labels import ( - db_get_active_label_embeddings, - db_write_image_semantic_scores, - ) + from app.config.settings import SEMANTIC_SCORE_TOP_K + from app.database.semantic_labels import db_write_image_semantic_scores from app.database.image_embeddings import db_get_embeddings_needing_scoring try: - metadata = SIGLIP2_SCORING_METADATA[SIGLIP2_ACTIVE_CHECKPOINT] - model_version = metadata["model_version"] - logit_scale = np.exp(metadata["logit_scale"]) - logit_bias = metadata["logit_bias"] - - meta, label_matrix = db_get_active_label_embeddings(model_version) - if not meta: + context = _load_label_scoring_context() + if context is None: logger.info( "No cached label embeddings for the active checkpoint; " "skipping semantic scoring pass" ) return - thresholds = np.array( - [ - ( - override - if override is not None - else SEMANTIC_BUCKET_THRESHOLDS.get( - category, SEMANTIC_DEFAULT_THRESHOLD - ) - ) - for _, category, override in meta - ], - dtype=np.float32, - ) - signature = _scoring_signature( - model_version, SEMANTIC_SCORE_TOP_K, thresholds, meta, label_matrix - ) + model_version = context["model_version"] + meta = context["meta"] + label_matrix = context["label_matrix"] + thresholds = context["thresholds"] + signature = context["signature"] + logit_scale = context["logit_scale"] + logit_bias = context["logit_bias"] total_images = 0 start_time = time.time() @@ -220,3 +248,94 @@ def semantic_util_score_images() -> None: ) except Exception as e: logger.error(f"Error in semantic scoring pass: {e}") + + +def semantic_util_score_videos() -> None: + """Score sampled keyframes and roll them up into video-level tags. + + Loops per video rather than in flat chunks like the image pass: a video's + tags are an aggregate over all its frames, so they have to be scored + together. A label's video score is its best frame's score, and it only + counts if enough frames agreed. + """ + import time + import numpy as np + from app.config.settings import VIDEO_TAG_MIN_FRAME_SUPPORT, VIDEO_TAG_TOP_K + from app.database.video_frames import ( + db_get_frame_embeddings_for_video, + db_get_videos_needing_scoring, + db_write_video_semantic_scores, + ) + + try: + context = _load_label_scoring_context() + if context is None: + logger.info( + "No cached label embeddings for the active checkpoint; " + "skipping video scoring pass" + ) + return + + model_version = context["model_version"] + meta = context["meta"] + label_matrix = context["label_matrix"] + thresholds = context["thresholds"] + signature = context["signature"] + logit_scale = context["logit_scale"] + logit_bias = context["logit_bias"] + + total_videos = 0 + start_time = time.time() + while True: + video_ids = db_get_videos_needing_scoring( + model_version, signature, SCORING_CHUNK_SIZE + ) + if not video_ids: + break + + for video_id in video_ids: + frame_matrix = db_get_frame_embeddings_for_video( + video_id, model_version + ) + if frame_matrix.size == 0: + # Can't happen given the query join, but stamping anyway + # keeps the while loop from spinning on such a row. + db_write_video_semantic_scores(video_id, [], signature) + continue + + logits = frame_matrix @ label_matrix.T * logit_scale + logit_bias + scores = 1.0 / (1.0 + np.exp(-logits)) # [frames, labels] + + hits = scores >= thresholds + frame_counts = hits.sum(axis=0) + best_scores = np.where(hits, scores, 0.0).max(axis=0) + + frame_total = frame_matrix.shape[0] + required = ( + VIDEO_TAG_MIN_FRAME_SUPPORT + if frame_total >= VIDEO_TAG_MIN_FRAME_SUPPORT + 1 + else 1 + ) + candidates = np.flatnonzero(frame_counts >= required) + if len(candidates) > VIDEO_TAG_TOP_K: + keep = np.argsort(best_scores[candidates])[::-1][:VIDEO_TAG_TOP_K] + candidates = candidates[keep] + + db_write_video_semantic_scores( + video_id, + [ + (meta[j][0], float(best_scores[j]), int(frame_counts[j])) + for j in candidates + ], + signature, + ) + total_videos += 1 + + if total_videos: + elapsed = time.time() - start_time + logger.info( + f"Video semantic scoring pass complete. Videos: {total_videos}, " + f"Labels: {len(meta)}, Elapsed: {elapsed:.2f}s" + ) + except Exception as e: + logger.error(f"Error in video semantic scoring pass: {e}") diff --git a/backend/app/utils/videos.py b/backend/app/utils/videos.py index 83930205c..68ce30193 100644 --- a/backend/app/utils/videos.py +++ b/backend/app/utils/videos.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import shutil import uuid import datetime import json @@ -11,7 +12,7 @@ import cv2 from PIL import Image -from app.config.settings import THUMBNAIL_IMAGES_PATH +from app.config.settings import THUMBNAIL_IMAGES_PATH, VIDEO_FRAMES_PATH from app.database.videos import ( VideoRecord, db_bulk_insert_videos, @@ -390,6 +391,352 @@ def video_util_remove_obsolete_videos(folder_id_list: List[int]) -> int: # Drop the rows first: an orphaned file beats a row pointing at nothing db_delete_videos_by_ids(obsolete_videos) video_util_remove_thumbnail_files(obsolete_thumbnails) + # The video_frames rows cascade with the video; their JPEGs don't. + for video_id in obsolete_videos: + video_util_remove_frame_directory(video_id) logger.info(f"Removed {len(obsolete_videos)} obsolete video(s) from database") return len(obsolete_videos) + + +# ============================================================================ +# KEYFRAME SAMPLING - AI tagging without per-frame inference +# ============================================================================ + + +def video_util_frame_directory(video_id: str) -> str: + """Where a video's sampled keyframe JPEGs live.""" + return os.path.join(VIDEO_FRAMES_PATH, video_id) + + +def video_util_remove_frame_directory(video_id: str) -> None: + frame_dir = video_util_frame_directory(video_id) + if os.path.isdir(frame_dir): + try: + shutil.rmtree(frame_dir) + except OSError as e: + logger.error(f"Error removing frame directory {frame_dir}: {e}") + + +def video_util_get_frame_interval() -> float: + """The user's keyframe interval, falling back to the configured default.""" + from app.config.settings import VIDEO_FRAME_INTERVAL_SECONDS + from app.database.metadata import db_get_metadata + + try: + metadata = db_get_metadata() or {} + interval = metadata.get("user_preferences", {}).get("Video_Frame_Interval") + if interval is not None: + return float(interval) + except Exception as e: + logger.warning(f"Could not read frame interval preference: {e}") + + return VIDEO_FRAME_INTERVAL_SECONDS + + +def video_util_sample_frame_timestamps( + duration: Optional[float], interval: float, max_frames: int +) -> List[float]: + """Timestamps (seconds) to sample a video at. + + Frames land at chunk midpoints -- the start of a video is often a black + leader frame, and a midpoint is the most representative moment of the + chunk it stands for. Videos long enough to exceed max_frames get their + interval stretched rather than their frame count grown, so inference cost + per video is bounded regardless of length. + """ + if not duration or duration <= 0 or interval <= 0: + return [0.0] + + effective_interval = max(interval, duration / max_frames) + if duration <= effective_interval: + return [duration / 2] + + timestamps = [] + position = effective_interval / 2 + while position < duration and len(timestamps) < max_frames: + timestamps.append(round(position, 3)) + position += effective_interval + + return timestamps or [duration / 2] + + +def video_util_extract_video_frames( + video_id: str, video_path: str, interval: float +) -> List[dict]: + """Sample a video into keyframe JPEGs on disk. + + Seeks to each timestamp rather than decoding sequentially -- that's what + keeps a two-hour video affordable. Returns frame records ready for + db_bulk_insert_video_frames; a frame that won't decode is skipped, not + fatal. + """ + from app.config.settings import ( + VIDEO_FRAME_MAX_DIMENSION, + VIDEO_MAX_FRAMES_PER_VIDEO, + ) + + frame_dir = video_util_frame_directory(video_id) + # Start from a clean sample: a re-tag must not mix frames from a previous + # interval into the new set. + video_util_remove_frame_directory(video_id) + + capture = cv2.VideoCapture(video_path) + try: + if not capture.isOpened(): + logger.warning(f"Could not open video for frame sampling: {video_path}") + return [] + + fps = capture.get(cv2.CAP_PROP_FPS) or 0 + frame_count = capture.get(cv2.CAP_PROP_FRAME_COUNT) or 0 + duration = frame_count / fps if fps > 0 and frame_count > 0 else None + + timestamps = video_util_sample_frame_timestamps( + duration, interval, VIDEO_MAX_FRAMES_PER_VIDEO + ) + + os.makedirs(frame_dir, exist_ok=True) + + frame_records = [] + for index, timestamp in enumerate(timestamps): + capture.set(cv2.CAP_PROP_POS_MSEC, timestamp * 1000) + ret, frame = capture.read() + if not ret or frame is None: + continue + + frame_path = os.path.abspath( + os.path.join(frame_dir, f"frame_{index:04d}.jpg") + ) + try: + img = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) + img.thumbnail((VIDEO_FRAME_MAX_DIMENSION, VIDEO_FRAME_MAX_DIMENSION)) + img.save(frame_path, "JPEG", quality=85) + except Exception as e: + logger.error(f"Error saving frame {index} of {video_path}: {e}") + continue + + frame_records.append( + { + "id": str(uuid.uuid4()), + "video_id": video_id, + "frame_path": frame_path, + "timestamp_sec": timestamp, + "frame_index": index, + } + ) + + return frame_records + except Exception as e: + logger.error(f"Error sampling frames for {video_path}: {e}") + return [] + finally: + capture.release() + + +def video_util_aggregate_frame_classes( + frame_class_ids: List[List[int]], min_support: int +) -> List[Tuple[int, int]]: + """Roll per-frame detections up to video-level (class_id, frame_count). + + A class is only kept if it was seen in enough frames -- one detection in + one unlucky keyframe describes the frame, not the video. Videos too short + to have that many frames fall back to a support of 1. + """ + if not frame_class_ids: + return [] + + counts: Dict[int, int] = {} + for class_ids in frame_class_ids: + for class_id in set(class_ids or []): + counts[class_id] = counts.get(class_id, 0) + 1 + + required = min_support if len(frame_class_ids) >= min_support + 1 else 1 + return sorted( + ((class_id, count) for class_id, count in counts.items() if count >= required), + key=lambda pair: (-pair[1], pair[0]), + ) + + +def video_util_process_untagged_videos() -> bool: + """Sample and object-tag every untagged video in AI-tagging folders.""" + from app.config.settings import VIDEO_TAG_MIN_FRAME_SUPPORT + from app.database.video_frames import ( + db_bulk_insert_video_frames, + db_delete_frames_for_videos, + db_get_untagged_videos, + db_mark_videos_tagged, + db_write_video_classes, + ) + from app.models.ObjectClassifier import ObjectClassifier + + try: + untagged_videos = db_get_untagged_videos() + if not untagged_videos: + return True + + interval = video_util_get_frame_interval() + object_classifier = ObjectClassifier() + total_frames = 0 + + try: + for video in untagged_videos: + video_id = video["id"] + frames = video_util_extract_video_frames( + video_id, video["path"], interval + ) + + if not frames: + # Undecodable files are still marked tagged, otherwise + # every folder sync retries them forever. + logger.warning( + f"No frames sampled from {video['path']}; tagging as empty" + ) + db_mark_videos_tagged([video_id]) + continue + + db_delete_frames_for_videos([video_id]) + db_bulk_insert_video_frames(frames) + total_frames += len(frames) + + frame_class_ids = [ + object_classifier.get_classes(frame["frame_path"]) or [] + for frame in frames + ] + db_write_video_classes( + video_id, + video_util_aggregate_frame_classes( + frame_class_ids, VIDEO_TAG_MIN_FRAME_SUPPORT + ), + ) + db_mark_videos_tagged([video_id]) + finally: + object_classifier.close() + + logger.info( + f"Video tagging pass complete. Videos: {len(untagged_videos)}, " + f"Frames sampled: {total_frames}, Interval: {interval}s" + ) + return True + except Exception as e: + logger.error(f"Error processing untagged videos: {e}") + return False + + +def video_util_process_unembedded_frames() -> None: + """Embed sampled keyframes with SigLIP2 -- the video counterpart of + image_util_process_unembedded_images.""" + import time + + import numpy as np + + from app.config.settings import ( + SIGLIP2_ACTIVE_CHECKPOINT, + SIGLIP2_SCORING_METADATA, + SIGLIP2_EMBED_BATCH_SIZE, + ) + from app.database.video_frames import ( + db_get_unembedded_video_frames, + db_mark_video_frames_embedded, + db_upsert_video_frame_embeddings, + ) + from app.models.model_registry import get_siglip2_registry_keys, get_model_path + from app.models.SigLIP2Vision import SigLIP2Vision + from app.utils.SigLIP import siglip_util_preprocess_image + + try: + vision_key, _ = get_siglip2_registry_keys(SIGLIP2_ACTIVE_CHECKPOINT) + vision_model_path = get_model_path(vision_key) + if not os.path.exists(vision_model_path): + logger.info( + "SigLIP2 vision model not installed; skipping frame embedding pass" + ) + return + + unembedded_frames = db_get_unembedded_video_frames() + if not unembedded_frames: + return + + metadata = SIGLIP2_SCORING_METADATA[SIGLIP2_ACTIVE_CHECKPOINT] + resolution = metadata["input_resolution"] + model_version = metadata["model_version"] + + vision_model = SigLIP2Vision(vision_model_path) + try: + total_frames = len(unembedded_frames) + embedded_count = 0 + corrupt_count = 0 + start_time = time.time() + + for i in range(0, total_frames, SIGLIP2_EMBED_BATCH_SIZE): + batch = unembedded_frames[i : i + SIGLIP2_EMBED_BATCH_SIZE] + + good_arrays = [] + good_ids = [] + + for frame in batch: + preprocessed = siglip_util_preprocess_image( + frame["frame_path"], resolution + ) + if preprocessed is None: + corrupt_count += 1 + continue + + good_arrays.append(preprocessed) + good_ids.append(frame["id"]) + + if not good_arrays: + continue + + stacked = np.stack(good_arrays) # [N, 3, R, R] + embeddings = vision_model.get_embedding(stacked) # [N, D] + + db_upsert_video_frame_embeddings( + [ + (good_ids[idx], model_version, emb) + for idx, emb in enumerate(embeddings) + ] + ) + embedded_count += len(good_arrays) + + # Only frames that actually got an embedding row, so a frame + # that fails to decode is retried next pass. + db_mark_video_frames_embedded(good_ids) + + elapsed = time.time() - start_time + logger.info( + f"Video frame embedding pass complete. Total: {total_frames}, " + f"Embedded: {embedded_count}, Corrupt: {corrupt_count}, " + f"Elapsed: {elapsed:.2f}s" + ) + finally: + vision_model.close() + + except Exception as e: + logger.error(f"Error embedding video frames: {e}") + + +def video_util_purge_frame_cache() -> int: + """Delete the keyframe JPEGs and return the bytes reclaimed. + + Frame rows and their embeddings survive, so tags and semantic search keep + working -- only a re-tag would need the frames extracted again. + """ + from app.database.video_frames import db_clear_frame_paths + + reclaimed = 0 + if os.path.isdir(VIDEO_FRAMES_PATH): + for root, _, files in os.walk(VIDEO_FRAMES_PATH): + for file in files: + try: + reclaimed += os.path.getsize(os.path.join(root, file)) + except OSError: + continue + try: + shutil.rmtree(VIDEO_FRAMES_PATH) + except OSError as e: + logger.error(f"Error purging frame cache: {e}") + return 0 + + db_clear_frame_paths() + logger.info(f"Purged video frame cache, reclaimed {reclaimed} bytes") + return reclaimed diff --git a/backend/main.py b/backend/main.py index a36375f66..a6eb64327 100644 --- a/backend/main.py +++ b/backend/main.py @@ -24,10 +24,12 @@ from app.database.metadata import db_create_metadata_table from app.database.semantic_labels import db_create_semantic_labels_table from app.database.image_embeddings import db_create_image_embeddings_table +from app.database.video_frames import db_create_video_frames_tables from app.utils.semantic_labels import ( semantic_util_sync_vocabulary, semantic_util_build_label_embeddings, semantic_util_score_images, + semantic_util_score_videos, ) from app.routes.folders import router as folders_router @@ -67,6 +69,7 @@ async def lifespan(app: FastAPI): db_create_videos_table() db_create_semantic_labels_table() db_create_image_embeddings_table() + db_create_video_frames_tables() db_create_YOLO_classes_table() db_create_clusters_table() # Create clusters table first since faces references it db_create_faces_table() @@ -83,6 +86,7 @@ async def lifespan(app: FastAPI): # order: the scoring sweep needs the label embeddings. app.state.executor.submit(semantic_util_build_label_embeddings) app.state.executor.submit(semantic_util_score_images) + app.state.executor.submit(semantic_util_score_videos) # Start the SSE model download cleanup task cleanup_task = asyncio.create_task(_cleanup_stale_tasks()) diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 964977f87..72cce0cf0 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -12,6 +12,7 @@ from app.database.metadata import db_create_metadata_table from app.database.semantic_labels import db_create_semantic_labels_table from app.database.image_embeddings import db_create_image_embeddings_table +from app.database.video_frames import db_create_video_frames_tables @pytest.fixture(scope="session", autouse=True) @@ -34,6 +35,7 @@ def setup_before_all_tests(): db_create_videos_table() db_create_semantic_labels_table() db_create_image_embeddings_table() + db_create_video_frames_tables() db_create_metadata_table() print("All database tables created successfully") except Exception as e: diff --git a/backend/tests/test_folders.py b/backend/tests/test_folders.py index 2ed2281ca..18585de83 100644 --- a/backend/tests/test_folders.py +++ b/backend/tests/test_folders.py @@ -36,6 +36,7 @@ db_find_parent_folder_id, ) from app.database.images import db_create_images_table +from app.database.videos import db_create_videos_table from app.database.yolo_mapping import db_create_YOLO_classes_table # ############################## @@ -52,6 +53,7 @@ def test_db(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]: monkeypatch.setattr("app.config.settings.DATABASE_PATH", db_path) monkeypatch.setattr("app.database.folders.DATABASE_PATH", db_path) monkeypatch.setattr("app.database.images.DATABASE_PATH", db_path) + monkeypatch.setattr("app.database.videos.DATABASE_PATH", db_path) monkeypatch.setattr("app.database.yolo_mapping.DATABASE_PATH", db_path) # Build the real schema rather than a hand-written copy: a divergent @@ -61,6 +63,7 @@ def test_db(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]: db_create_YOLO_classes_table() db_create_folders_table() db_create_images_table() # db_get_all_folder_details LEFT JOINs it + db_create_videos_table() # db_get_all_folder_details LEFT JOINs it too yield db_path finally: @@ -135,6 +138,7 @@ def sample_folder_details(): False, # taggingCompleted "completed", # indexing_status 0, # image_count + 7, # video_count -- a video-only folder is not "empty" ), ( "folder-id-2", @@ -145,6 +149,7 @@ def sample_folder_details(): True, "completed", 25, + 0, ), ] @@ -646,6 +651,9 @@ def test_get_all_folders_success( assert first_folder["parent_folder_id"] is None assert first_folder["AI_Tagging"] is True assert first_folder["taggingCompleted"] is False + # The video-only fixture folder must surface its media counts. + assert first_folder["image_count"] == 0 + assert first_folder["video_count"] == 7 mock_get_all_folders.assert_called_once() @@ -1122,9 +1130,52 @@ def test_db_get_all_folder_details(self, test_db): 0, # taggingCompleted "not_started", # indexing_status (schema default) 2, # image_count + 0, # video_count ) ] + def test_db_get_all_folder_details_counts_videos(self, test_db): + """Distinct image/video counts on both sides of the media join. + + The mixed folder deliberately has >1 of each: the images x videos + join fans out to 2*2=4 rows, so counting anything but DISTINCT ids + would over-report. A separate video-only folder pins image_count=0. + """ + conn = sqlite3.connect(test_db) + conn.executemany( + """ + INSERT INTO folders (folder_id, folder_path, parent_folder_id, + last_modified_time, AI_Tagging, taggingCompleted) + VALUES (?, ?, ?, ?, ?, ?) + """, + [ + ("mixed", "/home/user/media", None, 1693526400, True, False), + ("videos-only", "/home/user/clips", None, 1693526400, True, False), + ], + ) + conn.executemany( + "INSERT INTO images (id, path, folder_id) VALUES (?, ?, ?)", + [ + ("img-1", "/home/user/media/a.jpg", "mixed"), + ("img-2", "/home/user/media/b.jpg", "mixed"), + ], + ) + conn.executemany( + "INSERT INTO videos (id, path, folder_id) VALUES (?, ?, ?)", + [ + ("vid-1", "/home/user/media/a.mp4", "mixed"), + ("vid-2", "/home/user/media/b.mp4", "mixed"), + ("vid-3", "/home/user/clips/c.mp4", "videos-only"), + ], + ) + conn.commit() + conn.close() + + # (image_count, video_count) keyed by folder_id, order-independent. + counts = {row[0]: (row[7], row[8]) for row in db_get_all_folder_details()} + assert counts["mixed"] == (2, 2) + assert counts["videos-only"] == (0, 1) + def test_db_get_direct_child_folders(self, test_db): conn = sqlite3.connect(test_db) conn.execute( diff --git a/backend/tests/test_user_preferences.py b/backend/tests/test_user_preferences.py index e77ec15f4..911a956a4 100644 --- a/backend/tests/test_user_preferences.py +++ b/backend/tests/test_user_preferences.py @@ -504,3 +504,62 @@ def test_unsupported_http_methods(self, method, endpoint): """Test that unsupported HTTP methods return 405.""" response = client.request(method, endpoint) assert response.status_code == 405 + + +class TestVideoFrameIntervalValidation: + """Boundary coverage for the video keyframe interval on both preference + models, guarding the sampling API contract.""" + + def test_default_matches_sampler_config(self): + from app.schemas.user_preferences import UserPreferencesData + from app.config.settings import VIDEO_FRAME_INTERVAL_SECONDS + + assert ( + UserPreferencesData().Video_Frame_Interval == VIDEO_FRAME_INTERVAL_SECONDS + ) + + @pytest.mark.parametrize( + "value", + [ + 0.5, # exact minimum + 300.0, # exact maximum + 5.0, + ], + ) + def test_accepts_in_range_values(self, value): + from app.schemas.user_preferences import ( + UserPreferencesData, + UpdateUserPreferencesRequest, + ) + + assert UserPreferencesData(Video_Frame_Interval=value).Video_Frame_Interval == ( + value + ) + assert ( + UpdateUserPreferencesRequest( + Video_Frame_Interval=value + ).Video_Frame_Interval + == value + ) + + @pytest.mark.parametrize("value", [0.4, 300.1, 0.0, -1.0]) + def test_rejects_out_of_range_values(self, value): + from pydantic import ValidationError + from app.schemas.user_preferences import ( + UserPreferencesData, + UpdateUserPreferencesRequest, + ) + + with pytest.raises(ValidationError): + UserPreferencesData(Video_Frame_Interval=value) + with pytest.raises(ValidationError): + UpdateUserPreferencesRequest(Video_Frame_Interval=value) + + def test_partial_update_allows_none(self): + from app.schemas.user_preferences import UpdateUserPreferencesRequest + + assert UpdateUserPreferencesRequest().Video_Frame_Interval is None + assert ( + UpdateUserPreferencesRequest(Video_Frame_Interval=None).Video_Frame_Interval + is None + ) diff --git a/backend/tests/test_video_frames.py b/backend/tests/test_video_frames.py new file mode 100644 index 000000000..93a6131bc --- /dev/null +++ b/backend/tests/test_video_frames.py @@ -0,0 +1,448 @@ +import json +import os +import sqlite3 +import tempfile +import shutil + +import cv2 +import numpy as np +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from app.database.video_frames import ( + db_bulk_insert_video_frames, + db_clear_frame_paths, + db_create_video_frames_tables, + db_get_all_frame_embeddings, + db_get_frame_embeddings_for_video, + db_get_unembedded_video_frames, + db_get_untagged_videos, + db_get_video_ids_by_tag, + db_get_video_tags, + db_get_videos_needing_scoring, + db_mark_video_frames_embedded, + db_mark_videos_tagged, + db_upsert_video_frame_embeddings, + db_write_video_classes, + db_write_video_semantic_scores, +) +from app.database.videos import db_bulk_insert_videos, db_create_videos_table +from app.routes.videos import router as videos_router +from app.utils.videos import ( + video_util_aggregate_frame_classes, + video_util_extract_video_frames, + video_util_purge_frame_cache, + video_util_sample_frame_timestamps, +) + +# ############################## +# Pytest Fixtures +# ############################## + + +@pytest.fixture(scope="function") +def test_db(monkeypatch): + """Point every module that touches these tables at a fresh tempfile + database. video_frames borrows images._connect, so that module's + DATABASE_PATH has to be patched too.""" + db_fd, db_path = tempfile.mkstemp() + os.close(db_fd) + + monkeypatch.setattr("app.config.settings.DATABASE_PATH", db_path) + monkeypatch.setattr("app.database.images.DATABASE_PATH", db_path) + monkeypatch.setattr("app.database.videos.DATABASE_PATH", db_path) + monkeypatch.setattr("app.database.folders.DATABASE_PATH", db_path) + monkeypatch.setattr("app.database.yolo_mapping.DATABASE_PATH", db_path) + + from app.database.folders import db_create_folders_table + from app.database.yolo_mapping import db_create_YOLO_classes_table + + db_create_folders_table() + db_create_YOLO_classes_table() + db_create_videos_table() + db_create_video_frames_tables() + + yield db_path + + os.unlink(db_path) + + +@pytest.fixture +def frames_dir(monkeypatch): + """Redirect the frame cache at a temp dir so tests never touch the real + user data directory.""" + temp_dir = tempfile.mkdtemp() + monkeypatch.setattr("app.utils.videos.VIDEO_FRAMES_PATH", temp_dir) + yield temp_dir + shutil.rmtree(temp_dir, ignore_errors=True) + + +@pytest.fixture +def tagging_folder_id(test_db): + """A folder with AI tagging enabled, so its videos are pickable.""" + folder_id = "folder-ai-on" + conn = sqlite3.connect(test_db) + conn.execute( + "INSERT INTO folders (folder_id, folder_path, last_modified_time, AI_Tagging) " + "VALUES (?, ?, 0, 1)", + (folder_id, os.path.abspath("tagging-folder")), + ) + conn.commit() + conn.close() + return folder_id + + +@pytest.fixture +def video_id(tagging_folder_id): + video = { + "id": "vid-1", + "path": os.path.abspath("tagging-folder/clip.mp4"), + "folder_id": tagging_folder_id, + "thumbnailPath": "/thumbs/thumbnail_vid-1.jpg", + "metadata": json.dumps( + { + "name": "clip.mp4", + "date_created": "2026-01-01T00:00:00", + "width": 1920, + "height": 1080, + "duration": 30.0, + "fps": 30.0, + "file_location": "clip.mp4", + "file_size": 1024, + "item_type": "video/mp4", + } + ), + "isTagged": False, + "captured_at": "2026-01-01T00:00:00", + } + db_bulk_insert_videos([video]) + return video["id"] + + +@pytest.fixture +def temp_media_dir(): + temp_dir = tempfile.mkdtemp() + yield temp_dir + shutil.rmtree(temp_dir, ignore_errors=True) + + +@pytest.fixture +def real_video_file(temp_media_dir): + """A 3-second video: long enough that a 1s interval yields several frames.""" + video_path = os.path.join(temp_media_dir, "sample.avi") + writer = cv2.VideoWriter( + video_path, cv2.VideoWriter_fourcc(*"MJPG"), 10.0, (64, 64) + ) + if not writer.isOpened(): + pytest.skip("cv2.VideoWriter unavailable in this environment") + for i in range(30): + writer.write(np.full((64, 64, 3), (i * 8) % 256, dtype=np.uint8)) + writer.release() + return video_path + + +@pytest.fixture +def client(test_db): + app = FastAPI() + app.include_router(videos_router, prefix="/videos") + return TestClient(app) + + +def insert_frames(video_id, count, frames_dir=None): + """Insert `count` frame rows, one second apart.""" + records = [ + { + "id": f"{video_id}-frame-{i}", + "video_id": video_id, + "frame_path": os.path.join(frames_dir or "/frames", f"f{i}.jpg"), + "timestamp_sec": float(i), + "frame_index": i, + } + for i in range(count) + ] + db_bulk_insert_video_frames(records) + return records + + +# ############################## +# Sampling strategy +# ############################## + + +class TestSampleFrameTimestamps: + def test_samples_at_chunk_midpoints(self): + # 20s at 5s intervals: 2.5, 7.5, 12.5, 17.5 -- never the leading frame, + # which is often black. + assert video_util_sample_frame_timestamps(20.0, 5.0, 200) == [ + 2.5, + 7.5, + 12.5, + 17.5, + ] + + def test_ten_minute_video_at_five_seconds(self): + assert len(video_util_sample_frame_timestamps(600.0, 5.0, 200)) == 120 + + def test_long_video_stretches_interval_instead_of_frame_count(self): + timestamps = video_util_sample_frame_timestamps(3 * 3600.0, 5.0, 200) + assert len(timestamps) == 200 + # 3 hours over 200 frames = one frame per 54 seconds. + assert timestamps[1] - timestamps[0] == pytest.approx(54.0) + + def test_video_shorter_than_the_interval_yields_one_midpoint_frame(self): + assert video_util_sample_frame_timestamps(3.0, 5.0, 200) == [1.5] + + @pytest.mark.parametrize("duration", [None, 0, -1]) + def test_unknown_duration_falls_back_to_a_single_frame(self, duration): + assert video_util_sample_frame_timestamps(duration, 5.0, 200) == [0.0] + + def test_respects_the_cap_exactly(self): + assert len(video_util_sample_frame_timestamps(100.0, 1.0, 10)) == 10 + + +# ############################## +# Aggregating frames into video tags +# ############################## + + +class TestAggregateFrameClasses: + def test_drops_classes_seen_in_too_few_frames(self): + frames = [[0]] * 4 + [[15]] # person in 4 frames, cat in 1 + assert video_util_aggregate_frame_classes(frames, 2) == [(0, 4)] + + def test_counts_a_class_once_per_frame(self): + # Three people in one frame is still one frame of support. + frames = [[0, 0, 0], [0, 0]] + assert video_util_aggregate_frame_classes(frames, 2) == [(0, 2)] + + def test_short_videos_relax_the_support_requirement(self): + # Two frames can never reach a support of 2 for a class in one of them, + # so the requirement drops to 1 rather than tagging nothing. + assert video_util_aggregate_frame_classes([[0], [15]], 2) == [(0, 1), (15, 1)] + + def test_orders_by_frame_count_descending(self): + frames = [[0], [0, 15], [0, 15], [15]] + assert video_util_aggregate_frame_classes(frames, 2) == [(0, 3), (15, 3)] + + def test_no_frames_means_no_tags(self): + assert video_util_aggregate_frame_classes([], 2) == [] + + +# ############################## +# Frame extraction +# ############################## + + +class TestExtractVideoFrames: + def test_writes_a_jpeg_per_sampled_timestamp(self, real_video_file, frames_dir): + records = video_util_extract_video_frames("vid-1", real_video_file, 1.0) + + assert len(records) == 3 # 3 seconds of footage at 1s intervals + for record in records: + assert os.path.exists(record["frame_path"]) + assert record["video_id"] == "vid-1" + assert [r["frame_index"] for r in records] == [0, 1, 2] + + def test_frames_are_downscaled(self, real_video_file, frames_dir, monkeypatch): + monkeypatch.setattr("app.config.settings.VIDEO_FRAME_MAX_DIMENSION", 32) + records = video_util_extract_video_frames("vid-1", real_video_file, 1.0) + + frame = cv2.imread(records[0]["frame_path"]) + assert max(frame.shape[:2]) <= 32 + + def test_resampling_replaces_the_previous_frames(self, real_video_file, frames_dir): + dense = video_util_extract_video_frames("vid-1", real_video_file, 1.0) + sparse = video_util_extract_video_frames("vid-1", real_video_file, 10.0) + + assert len(sparse) < len(dense) + # The old dense sample is gone rather than left behind as orphans. + frame_dir = os.path.join(frames_dir, "vid-1") + assert len(os.listdir(frame_dir)) == len(sparse) + + def test_undecodable_file_yields_no_frames(self, temp_media_dir, frames_dir): + broken = os.path.join(temp_media_dir, "broken.mp4") + with open(broken, "wb") as f: + f.write(b"not a video") + + assert video_util_extract_video_frames("vid-1", broken, 5.0) == [] + + +# ############################## +# Database round-trips +# ############################## + + +class TestVideoFrameDatabase: + def test_untagged_videos_only_from_ai_tagging_folders(self, video_id): + assert [v["id"] for v in db_get_untagged_videos()] == [video_id] + + db_mark_videos_tagged([video_id]) + assert db_get_untagged_videos() == [] + + def test_frames_cascade_when_their_video_is_deleted(self, video_id, test_db): + insert_frames(video_id, 3) + + from app.database.videos import db_delete_videos_by_ids + + db_delete_videos_by_ids([video_id]) + + conn = sqlite3.connect(test_db) + count = conn.execute("SELECT COUNT(*) FROM video_frames").fetchone()[0] + conn.close() + assert count == 0 + + def test_embedded_frames_drop_out_of_the_pending_list(self, video_id): + frames = insert_frames(video_id, 3) + assert len(db_get_unembedded_video_frames()) == 3 + + db_mark_video_frames_embedded([frames[0]["id"]]) + assert len(db_get_unembedded_video_frames()) == 2 + + def test_purged_frames_are_not_re_embedded(self, video_id): + insert_frames(video_id, 3) + db_clear_frame_paths() + + # Nothing on disk to read, so they must not come back as pending work. + assert db_get_unembedded_video_frames() == [] + + def test_semantic_rescoring_leaves_yolo_tags_alone(self, video_id, test_db): + conn = sqlite3.connect(test_db) + conn.execute("INSERT INTO mappings (class_id, name) VALUES (1000, 'sunset')") + conn.commit() + conn.close() + + db_write_video_classes(video_id, [(0, 4)]) + db_write_video_semantic_scores(video_id, [(1000, 0.9, 3)], "sig-1") + assert sorted(db_get_video_tags([video_id])[video_id]) == [ + "person", + "sunset", + ] + + # A vocabulary change rewrites the semantic rows only. + db_write_video_semantic_scores(video_id, [], "sig-2") + assert db_get_video_tags([video_id])[video_id] == ["person"] + + def test_tag_lookup_is_case_insensitive(self, video_id): + db_write_video_classes(video_id, [(0, 4)]) + + assert db_get_video_ids_by_tag("PERSON") == [video_id] + assert db_get_video_ids_by_tag("giraffe") == [] + + def test_embeddings_round_trip_per_video_and_globally(self, video_id): + frames = insert_frames(video_id, 2) + db_upsert_video_frame_embeddings( + [ + (frames[0]["id"], "m1", np.array([1.0, 0.0], dtype=np.float32)), + (frames[1]["id"], "m1", np.array([0.0, 1.0], dtype=np.float32)), + ] + ) + + matrix = db_get_frame_embeddings_for_video(video_id, "m1") + assert matrix.shape == (2, 2) + + video_ids, timestamps, all_matrix = db_get_all_frame_embeddings("m1") + assert video_ids == [video_id, video_id] + assert timestamps == [0.0, 1.0] + assert all_matrix.shape == (2, 2) + + def test_scoring_signature_gates_rework(self, video_id): + frames = insert_frames(video_id, 1) + db_upsert_video_frame_embeddings( + [(frames[0]["id"], "m1", np.array([1.0, 0.0], dtype=np.float32))] + ) + + assert db_get_videos_needing_scoring("m1", "sig-1", 10) == [video_id] + + db_write_video_semantic_scores(video_id, [], "sig-1") + assert db_get_videos_needing_scoring("m1", "sig-1", 10) == [] + # A vocabulary change invalidates the stamp and the work comes back. + assert db_get_videos_needing_scoring("m1", "sig-2", 10) == [video_id] + + +# ############################## +# Purging the frame cache +# ############################## + + +class TestPurgeFrameCache: + def test_removes_jpegs_but_keeps_tags_and_embeddings( + self, video_id, real_video_file, frames_dir + ): + frames = video_util_extract_video_frames("vid-1", real_video_file, 1.0) + db_bulk_insert_video_frames(frames) + db_upsert_video_frame_embeddings( + [(frames[0]["id"], "m1", np.array([1.0, 0.0], dtype=np.float32))] + ) + db_write_video_classes(video_id, [(0, 3)]) + + reclaimed = video_util_purge_frame_cache() + + assert reclaimed > 0 + assert not os.path.exists(frames_dir) + # The index survives the cache: tags and semantic search keep working. + assert db_get_video_tags([video_id])[video_id] == ["person"] + assert db_get_frame_embeddings_for_video(video_id, "m1").shape == (1, 2) + + def test_purging_an_empty_cache_is_harmless(self, test_db, frames_dir): + shutil.rmtree(frames_dir, ignore_errors=True) + assert video_util_purge_frame_cache() == 0 + + +# ############################## +# Routes +# ############################## + + +class TestVideoTagRoutes: + def test_get_all_videos_returns_tags(self, client, video_id): + db_write_video_classes(video_id, [(0, 4)]) + + response = client.get("/videos/") + + assert response.status_code == 200 + assert response.json()["data"][0]["tags"] == ["person"] + + def test_untagged_video_reports_null_tags(self, client, video_id): + response = client.get("/videos/") + + assert response.json()["data"][0]["tags"] is None + + def test_search_by_tag(self, client, video_id): + db_write_video_classes(video_id, [(0, 4)]) + + response = client.get("/videos/search", params={"tag": "person"}) + + assert response.status_code == 200 + assert [v["id"] for v in response.json()["data"]] == [video_id] + + def test_search_by_tag_with_no_matches(self, client, video_id): + response = client.get("/videos/search", params={"tag": "person"}) + + assert response.status_code == 200 + assert response.json()["data"] == [] + + def test_semantic_search_without_models_returns_404( + self, client, video_id, monkeypatch + ): + monkeypatch.setattr( + "app.models.model_registry.get_model_path", + lambda key: "/nonexistent/model.onnx", + ) + + response = client.get("/videos/semantic-search", params={"query": "a beach"}) + + assert response.status_code == 404 + assert "not installed" in response.json()["detail"]["message"] + + def test_purge_route_reports_bytes_reclaimed( + self, client, video_id, real_video_file, frames_dir + ): + db_bulk_insert_video_frames( + video_util_extract_video_frames("vid-1", real_video_file, 1.0) + ) + + response = client.post("/videos/purge-frame-cache") + + assert response.status_code == 200 + assert response.json()["bytes_reclaimed"] > 0 diff --git a/docs/backend/backend_python/openapi.json b/docs/backend/backend_python/openapi.json index ede17f6e1..7aa2ade9d 100644 --- a/docs/backend/backend_python/openapi.json +++ b/docs/backend/backend_python/openapi.json @@ -990,7 +990,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SemanticSearchResponse" + "$ref": "#/components/schemas/app__routes__images__SemanticSearchResponse" } } } @@ -1110,6 +1110,179 @@ } } }, + "/videos/search": { + "get": { + "tags": [ + "Videos" + ], + "summary": "Search Videos By Tag", + "description": "Search videos by tag name.", + "operationId": "search_videos_by_tag_videos_search_get", + "parameters": [ + { + "name": "tag", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "Tag name to search for", + "title": "Tag" + }, + "description": "Tag name to search for" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GetAllVideosResponse" + } + } + } + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/app__schemas__videos__ErrorResponse" + } + } + }, + "description": "Bad Request" + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/app__schemas__videos__ErrorResponse" + } + } + }, + "description": "Internal Server Error" + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/videos/semantic-search": { + "get": { + "tags": [ + "Videos" + ], + "summary": "Semantic Search Videos", + "description": "Semantic search videos by query text using SigLIP2 keyframe embeddings.", + "operationId": "semantic_search_videos_videos_semantic_search_get", + "parameters": [ + { + "name": "query", + "in": "query", + "required": true, + "schema": { + "type": "string", + "minLength": 1, + "description": "Query text to search for", + "title": "Query" + }, + "description": "Query text to search for" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/app__routes__videos__SemanticSearchResponse" + } + } + } + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/app__schemas__videos__ErrorResponse" + } + } + }, + "description": "Bad Request" + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/app__schemas__videos__ErrorResponse" + } + } + }, + "description": "Not Found" + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/app__schemas__videos__ErrorResponse" + } + } + }, + "description": "Internal Server Error" + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/videos/purge-frame-cache": { + "post": { + "tags": [ + "Videos" + ], + "summary": "Purge Frame Cache", + "description": "Delete the sampled keyframe JPEGs. Tags and semantic search survive:\nonly the on-disk frames are removed, not their embeddings.", + "operationId": "purge_frame_cache_videos_purge_frame_cache_post", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PurgeFrameCacheResponse" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/app__schemas__videos__ErrorResponse" + } + } + } + } + } + } + }, "/videos/toggle-favourite": { "post": { "tags": [ @@ -2459,6 +2632,11 @@ "title": "Image Count", "default": 0 }, + "video_count": { + "type": "integer", + "title": "Video Count", + "default": 0 + }, "indexing_status": { "anyOf": [ { @@ -3309,6 +3487,29 @@ ], "title": "MultiPersonSearchResponse" }, + "PurgeFrameCacheResponse": { + "properties": { + "success": { + "type": "boolean", + "title": "Success" + }, + "message": { + "type": "string", + "title": "Message" + }, + "bytes_reclaimed": { + "type": "integer", + "title": "Bytes Reclaimed" + } + }, + "type": "object", + "required": [ + "success", + "message", + "bytes_reclaimed" + ], + "title": "PurgeFrameCacheResponse" + }, "RenameClusterData": { "properties": { "cluster_id": { @@ -3385,26 +3586,79 @@ ], "title": "RenameClusterResponse" }, - "SemanticSearchData": { + "ScoredVideoData": { "properties": { - "images": { - "items": { - "$ref": "#/components/schemas/SemanticSearchImage" - }, - "type": "array", - "title": "Images" + "id": { + "type": "string", + "title": "Id" }, - "total": { - "type": "integer", - "title": "Total" + "path": { + "type": "string", + "title": "Path" + }, + "folder_id": { + "type": "string", + "title": "Folder Id" + }, + "thumbnailPath": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Thumbnailpath" + }, + "metadata": { + "$ref": "#/components/schemas/VideoMetadataModel" + }, + "isFavourite": { + "type": "boolean", + "title": "Isfavourite" + }, + "tags": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Tags" + }, + "score": { + "type": "number", + "title": "Score" + }, + "best_frame_timestamp": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Best Frame Timestamp" } }, "type": "object", "required": [ - "images", - "total" + "id", + "path", + "folder_id", + "thumbnailPath", + "metadata", + "isFavourite", + "score" ], - "title": "SemanticSearchData" + "title": "ScoredVideoData" }, "SemanticSearchImage": { "properties": { @@ -3467,28 +3721,6 @@ ], "title": "SemanticSearchImage" }, - "SemanticSearchResponse": { - "properties": { - "success": { - "type": "boolean", - "title": "Success" - }, - "message": { - "type": "string", - "title": "Message" - }, - "data": { - "$ref": "#/components/schemas/SemanticSearchData" - } - }, - "type": "object", - "required": [ - "success", - "message", - "data" - ], - "title": "SemanticSearchResponse" - }, "SetupRequest": { "properties": { "tier": { @@ -3831,6 +4063,19 @@ } ], "title": "Gpu Acceleration" + }, + "Video_Frame_Interval": { + "anyOf": [ + { + "type": "number", + "maximum": 300.0, + "minimum": 0.5 + }, + { + "type": "null" + } + ], + "title": "Video Frame Interval" } }, "type": "object", @@ -3876,6 +4121,13 @@ "type": "boolean", "title": "Gpu Acceleration", "default": true + }, + "Video_Frame_Interval": { + "type": "number", + "maximum": 300.0, + "minimum": 0.5, + "title": "Video Frame Interval", + "default": 5.0 } }, "type": "object", @@ -4045,6 +4297,49 @@ ], "title": "VideoMetadataModel" }, + "app__routes__images__SemanticSearchData": { + "properties": { + "images": { + "items": { + "$ref": "#/components/schemas/SemanticSearchImage" + }, + "type": "array", + "title": "Images" + }, + "total": { + "type": "integer", + "title": "Total" + } + }, + "type": "object", + "required": [ + "images", + "total" + ], + "title": "SemanticSearchData" + }, + "app__routes__images__SemanticSearchResponse": { + "properties": { + "success": { + "type": "boolean", + "title": "Success" + }, + "message": { + "type": "string", + "title": "Message" + }, + "data": { + "$ref": "#/components/schemas/app__routes__images__SemanticSearchData" + } + }, + "type": "object", + "required": [ + "success", + "message", + "data" + ], + "title": "SemanticSearchResponse" + }, "app__routes__images__ToggleFavouriteRequest": { "properties": { "image_id": { @@ -4058,6 +4353,49 @@ ], "title": "ToggleFavouriteRequest" }, + "app__routes__videos__SemanticSearchData": { + "properties": { + "videos": { + "items": { + "$ref": "#/components/schemas/ScoredVideoData" + }, + "type": "array", + "title": "Videos" + }, + "total": { + "type": "integer", + "title": "Total" + } + }, + "type": "object", + "required": [ + "videos", + "total" + ], + "title": "SemanticSearchData" + }, + "app__routes__videos__SemanticSearchResponse": { + "properties": { + "success": { + "type": "boolean", + "title": "Success" + }, + "message": { + "type": "string", + "title": "Message" + }, + "data": { + "$ref": "#/components/schemas/app__routes__videos__SemanticSearchData" + } + }, + "type": "object", + "required": [ + "success", + "message", + "data" + ], + "title": "SemanticSearchResponse" + }, "app__routes__videos__ToggleFavouriteRequest": { "properties": { "video_id": { diff --git a/frontend/src/api/api-functions/user_preferences.ts b/frontend/src/api/api-functions/user_preferences.ts index bc0a02af7..391a01749 100644 --- a/frontend/src/api/api-functions/user_preferences.ts +++ b/frontend/src/api/api-functions/user_preferences.ts @@ -6,6 +6,8 @@ import { APIResponse } from '@/types/API'; export interface UserPreferencesData { YOLO_model_size: 'nano' | 'small' | 'medium'; GPU_Acceleration: boolean; + /** Seconds between sampled video keyframes when tagging videos. */ + Video_Frame_Interval: number; } export interface GetUserPreferencesResponse extends APIResponse { @@ -15,6 +17,7 @@ export interface GetUserPreferencesResponse extends APIResponse { export interface UpdateUserPreferencesRequest { YOLO_model_size?: 'nano' | 'small' | 'medium'; GPU_Acceleration?: boolean; + Video_Frame_Interval?: number; } export interface UpdateUserPreferencesResponse extends APIResponse { diff --git a/frontend/src/api/api-functions/videos.ts b/frontend/src/api/api-functions/videos.ts index 85fe3293d..a48e0ae5d 100644 --- a/frontend/src/api/api-functions/videos.ts +++ b/frontend/src/api/api-functions/videos.ts @@ -1,6 +1,7 @@ import { videosEndpoints } from '../apiEndpoints'; import { apiClient } from '../axiosConfig'; import { APIResponse } from '@/types/API'; +import { ScoredVideo } from '@/types/Media'; export const fetchAllVideos = async (): Promise => { const response = await apiClient.get( @@ -18,3 +19,50 @@ export const toggleVideoFav = async ( ); return response.data; }; + +export interface SearchVideosByTagRequest { + tag: string; +} + +export const searchVideosByTag = async ( + request: SearchVideosByTagRequest, +): Promise => { + const response = await apiClient.get( + videosEndpoints.searchByTag(request.tag), + ); + return response.data; +}; + +export interface SemanticSearchVideosRequest { + query: string; +} + +export interface SemanticSearchVideosAPIResponse extends APIResponse { + data?: { + videos: ScoredVideo[]; + total: number; + }; +} + +export const semanticSearchVideos = async ( + request: SemanticSearchVideosRequest, +): Promise => { + const response = await apiClient.get( + videosEndpoints.semanticSearch(request.query), + ); + return response.data; +}; + +export interface PurgeFrameCacheResponse { + success: boolean; + message: string; + bytes_reclaimed: number; +} + +export const purgeVideoFrameCache = + async (): Promise => { + const response = await apiClient.post( + videosEndpoints.purgeFrameCache, + ); + return response.data; + }; diff --git a/frontend/src/api/apiEndpoints.ts b/frontend/src/api/apiEndpoints.ts index 10a380902..b1b5a21a3 100644 --- a/frontend/src/api/apiEndpoints.ts +++ b/frontend/src/api/apiEndpoints.ts @@ -9,6 +9,10 @@ export const imagesEndpoints = { export const videosEndpoints = { getAllVideos: '/videos/', setFavourite: '/videos/toggle-favourite', + searchByTag: (tag: string) => `/videos/search?tag=${encodeURIComponent(tag)}`, + semanticSearch: (query: string) => + `/videos/semantic-search?query=${encodeURIComponent(query)}`, + purgeFrameCache: '/videos/purge-frame-cache', }; export const faceClustersEndpoints = { diff --git a/frontend/src/components/BackgroundTasks/LibraryProcessingIndicator.tsx b/frontend/src/components/BackgroundTasks/LibraryProcessingIndicator.tsx index 6896a5472..a8786cb7e 100644 --- a/frontend/src/components/BackgroundTasks/LibraryProcessingIndicator.tsx +++ b/frontend/src/components/BackgroundTasks/LibraryProcessingIndicator.tsx @@ -9,7 +9,7 @@ import { ROUTES } from '@/constants/routes'; interface Dismissal { phase: LibraryProcessingPhase; - totalImages: number; + totalItems: number; } /** @@ -17,17 +17,17 @@ interface Dismissal { * Hidden on Settings, which has its own bars. Needs router context. */ export const LibraryProcessingIndicator: React.FC = () => { - const { phase, percentage, totalImages, semanticAvailable } = + const { phase, percentage, totalItems, semanticAvailable } = useLibraryProcessingStatus(); const location = useLocation(); // Dismissal is per-phase and invalidated when the library grows, so the - // alert resurfaces when indexing starts or new photos appear. + // alert resurfaces when indexing starts or new media appears. const [dismissal, setDismissal] = useState(null); const dismissed = dismissal !== null && dismissal.phase === phase && - totalImages <= dismissal.totalImages; + totalItems <= dismissal.totalItems; const onSettingsPage = location.pathname.includes(`/${ROUTES.SETTINGS}`); @@ -36,15 +36,15 @@ export const LibraryProcessingIndicator: React.FC = () => { const step = (n: number) => (semanticAvailable ? `Step ${n} of 2: ` : ''); const description = phase === 'tagging' - ? `${step(1)}Looking through your photos to recognize what's in them.` - : 'Step 2 of 2: Making your photos searchable, so you can find them by describing them in your own words.'; + ? `${step(1)}Looking through your photos and videos to recognize what's in them.` + : 'Step 2 of 2: Making your photos and videos searchable, so you can find them by describing them in your own words.'; return ( setDismissal({ phase, totalImages })} + onDismiss={() => setDismissal({ phase, totalItems })} /> ); }; diff --git a/frontend/src/components/Media/ImageTags.tsx b/frontend/src/components/Media/ImageTags.tsx index bd0b0da79..7598a64b5 100644 --- a/frontend/src/components/Media/ImageTags.tsx +++ b/frontend/src/components/Media/ImageTags.tsx @@ -1,4 +1,5 @@ import { Badge } from '@/components/ui/badge'; +import { cn } from '@/lib/utils'; import { Tag } from 'lucide-react'; import { useState, useEffect, useRef } from 'react'; @@ -6,12 +7,15 @@ interface ImageTagsProps { tags: any[]; showTags?: boolean; isImageHovered: boolean; + /** Extra classes for the anchor, e.g. a z-index above overlay controls. */ + className?: string; } export function ImageTags({ tags, showTags = true, isImageHovered, + className, }: ImageTagsProps) { const [isTagsHovered, setIsTagsHovered] = useState(false); const [showTagsFromImageHover, setShowTagsFromImageHover] = useState(false); @@ -50,7 +54,10 @@ export function ImageTags({ return (
showTags && setIsTagsHovered(true)} onMouseLeave={() => showTags && setIsTagsHovered(false)} > diff --git a/frontend/src/components/Media/VideoCard.tsx b/frontend/src/components/Media/VideoCard.tsx index 222a3e55a..bab83879c 100644 --- a/frontend/src/components/Media/VideoCard.tsx +++ b/frontend/src/components/Media/VideoCard.tsx @@ -7,6 +7,7 @@ import { Video } from '@/types/Media'; import { convertFileSrc } from '@tauri-apps/api/core'; import { useToggleVideoFav } from '@/hooks/useToggleVideoFav'; import { formatDurationLabel } from '@/utils/durationUtils'; +import { ImageTags } from '@/components/Media/ImageTags'; interface VideoCardProps { video: Video; @@ -20,6 +21,7 @@ export function VideoCard({ video, className, onClick }: VideoCardProps) { // A rescan can replace the poster on disk, so fall back rather than showing // a broken image if the file is gone by the time the card renders. const [thumbnailFailed, setThumbnailFailed] = useState(false); + const [isHovered, setIsHovered] = useState(false); useEffect(() => setThumbnailFailed(false), [video.thumbnailPath]); @@ -35,6 +37,8 @@ export function VideoCard({ video, className, onClick }: VideoCardProps) { 'group bg-card overflow-hidden rounded-lg border transition-all hover:shadow-md', className, )} + onMouseEnter={() => setIsHovered(true)} + onMouseLeave={() => setIsHovered(false)} >
@@ -109,6 +113,14 @@ export function VideoCard({ video, className, onClick }: VideoCardProps) {
)} + + {/* Tags sampled from the video's keyframes. Stacked above the + playback overlay so the badge stays hoverable. */} +
); diff --git a/frontend/src/components/Media/__tests__/VideoCard.test.tsx b/frontend/src/components/Media/__tests__/VideoCard.test.tsx index 7e8e07350..324f6a21f 100644 --- a/frontend/src/components/Media/__tests__/VideoCard.test.tsx +++ b/frontend/src/components/Media/__tests__/VideoCard.test.tsx @@ -132,4 +132,22 @@ describe('VideoCard', () => { fireEvent.click(favourite); expect(mockToggleFavourite).not.toHaveBeenCalled(); }); + + test('shows a tag count badge for tagged videos, and the tags on hover', async () => { + render(); + + // Collapsed: just the count. + expect(screen.getByText('2')).toBeInTheDocument(); + expect(screen.queryByText('beach')).not.toBeInTheDocument(); + + fireEvent.mouseEnter(screen.getByText('2')); + expect(await screen.findByText('beach')).toBeInTheDocument(); + expect(screen.getByText('sunset')).toBeInTheDocument(); + }); + + test('renders no tag badge when the video has no tags', () => { + render(); + + expect(screen.queryByText('0')).not.toBeInTheDocument(); + }); }); diff --git a/frontend/src/components/VideoPlayer/VideoInfoPanel.tsx b/frontend/src/components/VideoPlayer/VideoInfoPanel.tsx index 8788cd8da..4bc67729e 100644 --- a/frontend/src/components/VideoPlayer/VideoInfoPanel.tsx +++ b/frontend/src/components/VideoPlayer/VideoInfoPanel.tsx @@ -1,6 +1,6 @@ import React from 'react'; import { motion, AnimatePresence } from 'framer-motion'; -import { X, Film, Calendar, Clock, Monitor, Info } from 'lucide-react'; +import { X, Film, Calendar, Clock, Monitor, Info, Tag } from 'lucide-react'; import { Video } from '@/types/Media'; import { formatDurationLabel } from '@/utils/durationUtils'; @@ -41,6 +41,7 @@ export const VideoInfoPanel: React.FC = ({ const height = video?.metadata?.height ?? 0; const resolution = width > 0 && height > 0 ? `${width} × ${height}` : 'Not available'; + const tags = video?.tags ?? []; return ( @@ -111,6 +112,27 @@ export const VideoInfoPanel: React.FC = ({ + {tags.length > 0 && ( +
+
+ +
+
+

Tags

+
+ {tags.map((tag, index) => ( + + {tag} + + ))} +
+
+
+ )} +
diff --git a/frontend/src/hooks/useLibraryProcessingStatus.ts b/frontend/src/hooks/useLibraryProcessingStatus.ts index d6baf30bc..2ac8ce8b2 100644 --- a/frontend/src/hooks/useLibraryProcessingStatus.ts +++ b/frontend/src/hooks/useLibraryProcessingStatus.ts @@ -14,9 +14,10 @@ export interface LibraryProcessingStatus { phase: LibraryProcessingPhase; /** Progress (0-100) of the current phase; 100 when idle */ percentage: number; - totalImages: number; - taggedImages: number; - embeddedImages: number; + /** Counts are over images and videos combined. */ + totalItems: number; + taggedItems: number; + embeddedItems: number; } const aggregate = (data: APIResponse | undefined) => { @@ -25,17 +26,18 @@ const aggregate = (data: APIResponse | undefined) => { ? (rawFolders as FolderTaggingInfo[]) : []; - let totalImages = 0; - let taggedImages = 0; - let embeddedImages = 0; + let totalItems = 0; + let taggedItems = 0; + let embeddedItems = 0; for (const folder of folders) { // Only AI-tagging folders are ever processed if (!folder?.ai_tagging) continue; - totalImages += folder.total_images ?? 0; - taggedImages += folder.tagged_images ?? 0; - embeddedImages += folder.embedded_images ?? 0; + totalItems += (folder.total_images ?? 0) + (folder.total_videos ?? 0); + taggedItems += (folder.tagged_images ?? 0) + (folder.tagged_videos ?? 0); + embeddedItems += + (folder.embedded_images ?? 0) + (folder.embedded_videos ?? 0); } - return { totalImages, taggedImages, embeddedImages }; + return { totalItems, taggedItems, embeddedItems }; }; /** @@ -59,12 +61,12 @@ export const useLibraryProcessingStatus = (): LibraryProcessingStatus => { queryFn: getFoldersTaggingStatus, staleTime: 1000, refetchInterval: (query) => { - const { totalImages, taggedImages, embeddedImages } = aggregate( + const { totalItems, taggedItems, embeddedItems } = aggregate( query.state.data, ); const busy = - taggedImages < totalImages || - (semanticAvailable && embeddedImages < totalImages); + taggedItems < totalItems || + (semanticAvailable && embeddedItems < totalItems); return busy ? 1000 : 10000; }, refetchIntervalInBackground: true, @@ -73,30 +75,30 @@ export const useLibraryProcessingStatus = (): LibraryProcessingStatus => { refetchOnWindowFocus: false, }); - const { totalImages, taggedImages, embeddedImages } = aggregate( + const { totalItems, taggedItems, embeddedItems } = aggregate( taggingStatusQuery.data, ); let phase: LibraryProcessingPhase = 'idle'; let percentage = 100; - if (totalImages > 0 && taggedImages < totalImages) { + if (totalItems > 0 && taggedItems < totalItems) { phase = 'tagging'; - percentage = (taggedImages / totalImages) * 100; + percentage = (taggedItems / totalItems) * 100; } else if ( semanticAvailable && - totalImages > 0 && - embeddedImages < totalImages + totalItems > 0 && + embeddedItems < totalItems ) { phase = 'indexing'; - percentage = (embeddedImages / totalImages) * 100; + percentage = (embeddedItems / totalItems) * 100; } return { semanticAvailable, phase, percentage, - totalImages, - taggedImages, - embeddedImages, + totalItems, + taggedItems, + embeddedItems, }; }; diff --git a/frontend/src/hooks/useUserPreferences.tsx b/frontend/src/hooks/useUserPreferences.tsx index e200fa2f8..483e3866a 100644 --- a/frontend/src/hooks/useUserPreferences.tsx +++ b/frontend/src/hooks/useUserPreferences.tsx @@ -15,6 +15,7 @@ export const useUserPreferences = () => { const [preferences, setPreferences] = useState({ YOLO_model_size: 'nano', GPU_Acceleration: false, + Video_Frame_Interval: 5, }); // Query for user preferences @@ -89,6 +90,17 @@ export const useUserPreferences = () => { return updatePreference(updatedPreferences); }; + /** + * Update the video keyframe sampling interval (seconds) + */ + const updateVideoFrameInterval = async (interval: number) => { + const updatedPreferences = { + ...preferences, + Video_Frame_Interval: interval, + }; + return updatePreference(updatedPreferences); + }; + return { // Data preferences, @@ -98,6 +110,7 @@ export const useUserPreferences = () => { updatePreference, updateYoloModelSize, toggleGpuAcceleration, + updateVideoFrameInterval, // For refetching preferences after external events (e.g., Model Manager window closing) refetch: preferencesQuery.refetch, diff --git a/frontend/src/pages/SearchResults/SearchResults.tsx b/frontend/src/pages/SearchResults/SearchResults.tsx index f4bb0fac5..a97a8c605 100644 --- a/frontend/src/pages/SearchResults/SearchResults.tsx +++ b/frontend/src/pages/SearchResults/SearchResults.tsx @@ -3,17 +3,27 @@ import { useDispatch, useSelector } from 'react-redux'; import { AxiosError } from 'axios'; import { ImageCard } from '@/components/Media/ImageCard'; import { MediaView } from '@/components/Media/MediaView'; -import { Image, ScoredImage } from '@/types/Media'; +import { VideoCard } from '@/components/Media/VideoCard'; +import { VideoPlayerOverlay } from '@/components/VideoPlayer/VideoPlayerOverlay'; +import { Image, ScoredImage, ScoredVideo, Video } from '@/types/Media'; import { setCurrentViewIndex, setImages } from '@/features/imageSlice'; +import { + setCurrentViewIndex as setCurrentVideoViewIndex, + setVideos, +} from '@/features/videoSlice'; import { showLoader, hideLoader } from '@/features/loaderSlice'; import { showInfoDialog } from '@/features/infoDialogSlice'; import { selectImages, selectIsImageViewOpen } from '@/features/imageSelectors'; +import { selectIsVideoViewOpen, selectVideos } from '@/features/videoSelectors'; import { usePictoQuery } from '@/hooks/useQueryExtension'; import { searchImagesByTag, semanticSearchImages, + searchVideosByTag, + semanticSearchVideos, fetchModelStatus, SemanticSearchAPIResponse, + SemanticSearchVideosAPIResponse, } from '@/api/api-functions'; import { APIResponse } from '@/types/API'; import { getErrorMessage } from '@/lib/utils'; @@ -33,6 +43,16 @@ interface SemanticSearchResult extends SemanticSearchAPIResponse { type SearchQueryResult = TagSearchResult | SemanticSearchResult; +interface VideoTagSearchResult extends APIResponse { + resultType: 'tag'; +} + +interface VideoSemanticSearchResult extends SemanticSearchVideosAPIResponse { + resultType: 'semantic'; +} + +type VideoSearchQueryResult = VideoTagSearchResult | VideoSemanticSearchResult; + const getHttpStatus = (error: unknown): number | undefined => { const axiosErr = error as AxiosError; return axiosErr?.isAxiosError ? axiosErr.response?.status : undefined; @@ -46,6 +66,8 @@ export const SearchResults = () => { const mode = searchParams.get('mode') || 'auto'; const isImageViewOpen = useSelector(selectIsImageViewOpen); const displayImages = useSelector(selectImages); + const isVideoViewOpen = useSelector(selectIsVideoViewOpen); + const displayVideos = useSelector(selectVideos); const [searchError, setSearchError] = useState(null); @@ -95,6 +117,76 @@ export const SearchResults = () => { enabled: !!query, }); + // Videos run as their own query: they share the mode logic but a video + // failure (e.g. no frames embedded yet) must not blank the image results, + // and vice versa. + const { + data: videoData, + isSuccess: isVideoSuccess, + isError: isVideoError, + error: videoError, + errorMessage: videoErrorMessage, + } = usePictoQuery({ + queryKey: ['search-results-videos', query, mode], + queryFn: async (): Promise => { + if (mode === 'semantic') { + const res = await semanticSearchVideos({ query }); + return { ...res, resultType: 'semantic' }; + } + + const tagResponse = await searchVideosByTag({ tag: query }); + if (mode === 'tag' || (tagResponse.data?.length ?? 0) > 0) { + return { ...tagResponse, resultType: 'tag' }; + } + + // Reuse the status already fetched above; only fetch again if it + // hasn't resolved yet, avoiding a redundant round-trip per search. + let semAvailable = semanticAvailable; + if (!isStatusSuccess) { + const statusRes = await fetchModelStatus(); + semAvailable = + statusRes.success && statusRes.data + ? isSemanticSearchAvailable(statusRes.data) + : false; + } + + if (semAvailable) { + const semResponse = await semanticSearchVideos({ query }); + return { ...semResponse, resultType: 'semantic' }; + } + + return { ...tagResponse, resultType: 'tag' }; + }, + enabled: !!query, + }); + + useEffect(() => { + if (!isVideoSuccess || !videoData) return; + + const fetchedVideos: Video[] = + videoData.resultType === 'semantic' + ? ((videoData.data?.videos ?? []) as ScoredVideo[]) + : ((videoData.data ?? []) as Video[]); + + dispatch(setVideos(fetchedVideos)); + }, [videoData, isVideoSuccess, dispatch]); + + // A failed video search should not be silent; surface it in the logs like + // the image query's error branch does. + useEffect(() => { + if (isVideoError) { + console.error( + 'Video search failed:', + getErrorMessage(videoError, videoErrorMessage), + ); + } + }, [isVideoError, videoError, videoErrorMessage]); + + const videoSearchError = isVideoError + ? getErrorMessage(videoError, videoErrorMessage) || + 'Failed to search videos' + : null; + const effectiveMode = data?.resultType || mode; useEffect(() => { @@ -176,7 +268,12 @@ export const SearchResults = () => {
)} - {searchError ? ( + {!query ? ( +
+

Please enter a search term to find photos and videos.

+
+ ) : searchError && videoSearchError ? ( + // Only a total failure (both media types) takes over the whole view.

@@ -184,35 +281,81 @@ export const SearchResults = () => {

{searchError}

- ) : !query ? ( -
-

Please enter a search term to find images.

-
- ) : displayImages.length === 0 && isSuccess ? ( -
- {effectiveMode === 'semantic' ? ( -

No matches found. Try describing the photo differently.

+ ) : ( + <> + {/* Photos: an image-search error is shown inline so it never hides + successfully-fetched videos. */} + {searchError ? ( +
+ + Couldn't load photo results: {searchError} +
) : ( -

No images found matching your search.

+ displayImages.length > 0 && ( +
+ {displayImages.map((image, index) => ( +
+ dispatch(setCurrentViewIndex(index))} + /> +
+ ))} +
+ ) )} -
- ) : ( -
- {displayImages.map((image, index) => ( -
- dispatch(setCurrentViewIndex(index))} - /> + + {/* Videos: symmetric — a video-search error is inline and never + hides successfully-fetched photos. */} + {videoSearchError ? ( +
+ + Couldn't load video results: {videoSearchError}
- ))} -
+ ) : ( + displayVideos.length > 0 && ( + <> +

Videos

+
+ {displayVideos.map((video, index) => ( +
+ + dispatch(setCurrentVideoViewIndex(index)) + } + /> +
+ ))} +
+ + ) + )} + + {/* Both searches succeeded but nothing matched. */} + {!searchError && + !videoSearchError && + isSuccess && + isVideoSuccess && + displayImages.length === 0 && + displayVideos.length === 0 && ( +
+ {effectiveMode === 'semantic' ? ( +

No matches found. Try describing it differently.

+ ) : ( +

No photos or videos found matching your search.

+ )} +
+ )} + )} - {/* Media Viewer Modal */} + {/* Media Viewer Modals */} {isImageViewOpen && } + {isVideoViewOpen && }
); }; diff --git a/frontend/src/pages/SettingsPage/components/FolderManagementCard.tsx b/frontend/src/pages/SettingsPage/components/FolderManagementCard.tsx index fa26967b1..17b33cc0a 100644 --- a/frontend/src/pages/SettingsPage/components/FolderManagementCard.tsx +++ b/frontend/src/pages/SettingsPage/components/FolderManagementCard.tsx @@ -101,7 +101,7 @@ const FolderManagementCard: React.FC = () => { Indexing Folder...
- ) : folder.image_count === 0 ? ( + ) : !folder.image_count && !folder.video_count ? (
Folder is empty
diff --git a/frontend/src/pages/SettingsPage/components/UserPreferencesCard.tsx b/frontend/src/pages/SettingsPage/components/UserPreferencesCard.tsx index 44d25cb80..2822163f9 100644 --- a/frontend/src/pages/SettingsPage/components/UserPreferencesCard.tsx +++ b/frontend/src/pages/SettingsPage/components/UserPreferencesCard.tsx @@ -1,5 +1,12 @@ import React, { useEffect, useCallback, useState } from 'react'; -import { Cpu, ChevronDown, Zap } from 'lucide-react'; +import { + Cpu, + ChevronDown, + Zap, + Trash2, + Clapperboard, + HardDrive, +} from 'lucide-react'; import { Label } from '@/components/ui/label'; import { Switch } from '@/components/ui/switch'; @@ -17,8 +24,9 @@ import { listen } from '@tauri-apps/api/event'; import { getCurrentWindow } from '@tauri-apps/api/window'; import { useUserPreferences } from '@/hooks/useUserPreferences'; +import { purgeVideoFrameCache } from '@/api/api-functions'; import SettingsCard from './SettingsCard'; -import { formatTierLabel } from '@/lib/utils'; +import { cn, formatTierLabel } from '@/lib/utils'; import { BACKEND_URL } from '@/config/Backend'; import { getInstalledModelTiers, @@ -30,12 +38,37 @@ import { /** * Component for managing user preferences in settings */ +// Coarse enough to be a meaningful cost tradeoff, fine enough to matter. +const FRAME_INTERVAL_OPTIONS = [2, 5, 10, 30]; + const UserPreferencesCard: React.FC = () => { - const { preferences, updateYoloModelSize, toggleGpuAcceleration, refetch } = - useUserPreferences(); + const { + preferences, + updateYoloModelSize, + toggleGpuAcceleration, + updateVideoFrameInterval, + refetch, + } = useUserPreferences(); const [installedTiers, setInstalledTiers] = useState([]); const [loadingTiers, setLoadingTiers] = useState(true); const [tierFetchError, setTierFetchError] = useState(null); + const [purgeState, setPurgeState] = useState<'idle' | 'purging' | 'done'>( + 'idle', + ); + // Collapsed by default: video tagging is a niche setting, so it stays out + // of the way until a user with videos goes looking for it. + const [videoSettingsOpen, setVideoSettingsOpen] = useState(false); + + const handlePurgeFrameCache = useCallback(async () => { + setPurgeState('purging'); + try { + await purgeVideoFrameCache(); + setPurgeState('done'); + } catch (err) { + console.error('Failed to purge video frame cache', err); + setPurgeState('idle'); + } + }, []); useEffect(() => { const controller = new AbortController(); @@ -219,6 +252,114 @@ const UserPreferencesCard: React.FC = () => { /> + + {/* Video Tagging: a collapsible group so these niche controls don't + clutter the panel for users who only have photos. */} +
+ + + {videoSettingsOpen && ( +
+ {/* Video Keyframe Interval Setting */} +
+
+ +

+ How often a frame is sampled from a video for tagging. + Shorter intervals catch more detail but take longer to + process. Applies to videos tagged from now on. +

+
+ + + + + + {FRAME_INTERVAL_OPTIONS.map((seconds) => ( + + updateVideoFrameInterval(seconds).catch(console.warn) + } + > + {seconds} seconds + + ))} + + +
+ + {/* Video Frame Cache */} +
+
+ +

+ Reclaim the disk space used by sampled video frames. + Existing tags and video search keep working. +

+
+
+ + +
+
+
+ )} +
); diff --git a/frontend/src/pages/__tests__/SearchResults.test.tsx b/frontend/src/pages/__tests__/SearchResults.test.tsx index fa85ae00d..043ce7e02 100644 --- a/frontend/src/pages/__tests__/SearchResults.test.tsx +++ b/frontend/src/pages/__tests__/SearchResults.test.tsx @@ -1,6 +1,6 @@ import { render, screen, waitFor } from '@/test-utils'; import { SearchResults } from '../SearchResults/SearchResults'; -import { searchImagesByTag } from '@/api/api-functions'; +import { searchImagesByTag, searchVideosByTag } from '@/api/api-functions'; // Mock the API functions. fetchModelStatus must resolve to a well-formed // response even in tests that don't care about it -- the "no tag matches" @@ -8,12 +8,23 @@ import { searchImagesByTag } from '@/api/api-functions'; jest.mock('@/api/api-functions', () => ({ searchImagesByTag: jest.fn(), semanticSearchImages: jest.fn(), + searchVideosByTag: jest.fn(), + semanticSearchVideos: jest.fn(), fetchModelStatus: jest.fn().mockResolvedValue({ success: true, data: {} }), })); +jest.mock('@tauri-apps/api/core', () => ({ + convertFileSrc: (path: string) => path, +})); + describe('SearchResults Page', () => { beforeEach(() => { jest.clearAllMocks(); + // Videos default to no matches; the tests that care override this. + (searchVideosByTag as jest.Mock).mockResolvedValue({ + success: true, + data: [], + }); }); const renderWithQuery = (queryValue: string) => { @@ -27,12 +38,13 @@ describe('SearchResults Page', () => { test('renders empty state when no query is provided', async () => { renderWithQuery(''); - // Should not call the API if query is empty + // Neither search runs when there's no query expect(searchImagesByTag).not.toHaveBeenCalled(); + expect(searchVideosByTag).not.toHaveBeenCalled(); // Check for empty state message expect( - screen.getByText(/Please enter a search term to find images/i), + screen.getByText(/Please enter a search term to find/i), ).toBeInTheDocument(); }); @@ -67,7 +79,7 @@ describe('SearchResults Page', () => { // Wait for images to render await waitFor(() => { expect( - screen.queryByText(/No images found matching your search/i), + screen.queryByText(/No photos or videos found matching your search/i), ).not.toBeInTheDocument(); expect( screen.queryByText(/Please enter a search term/i), @@ -75,7 +87,36 @@ describe('SearchResults Page', () => { }); }); - test('renders no images found state when API returns empty array', async () => { + test('renders matching videos in their own section', async () => { + (searchImagesByTag as jest.Mock).mockResolvedValue({ + success: true, + data: [], + }); + (searchVideosByTag as jest.Mock).mockResolvedValue({ + success: true, + data: [ + { + id: 'v1', + path: '/clip.mp4', + thumbnailPath: '/clip-thumb.jpg', + folder_id: '1', + tags: ['beach'], + metadata: { name: 'clip.mp4' }, + }, + ], + }); + + renderWithQuery('beach'); + + expect(searchVideosByTag).toHaveBeenCalledWith({ tag: 'beach' }); + + await waitFor(() => { + expect(screen.getByText('Videos')).toBeInTheDocument(); + expect(screen.getByLabelText(/Play clip.mp4/i)).toBeInTheDocument(); + }); + }); + + test('renders no results state when both APIs return empty arrays', async () => { (searchImagesByTag as jest.Mock).mockResolvedValue({ success: true, data: [], @@ -85,15 +126,87 @@ describe('SearchResults Page', () => { await waitFor(() => { expect( - screen.getByText(/No images found matching your search/i), + screen.getByText(/No photos or videos found matching your search/i), ).toBeInTheDocument(); }); }); - test('renders error state when API request fails', async () => { + test('a photo-search error is shown inline and does not hide video results', async () => { + (searchImagesByTag as jest.Mock).mockRejectedValue( + new Error('Network Error'), + ); + (searchVideosByTag as jest.Mock).mockResolvedValue({ + success: true, + data: [ + { + id: 'v1', + path: '/clip.mp4', + thumbnailPath: '/clip-thumb.jpg', + folder_id: '1', + tags: ['beach'], + metadata: { name: 'clip.mp4' }, + }, + ], + }); + + renderWithQuery('dog'); + + await waitFor( + () => { + // Inline photo error, not the full-screen takeover... + expect( + screen.getByText(/Couldn't load photo results/i), + ).toBeInTheDocument(); + expect(screen.queryByText(/^Search Failed$/i)).not.toBeInTheDocument(); + // ...and the videos still render. + expect(screen.getByText('Videos')).toBeInTheDocument(); + expect(screen.getByLabelText(/Play clip.mp4/i)).toBeInTheDocument(); + }, + { timeout: 3000 }, + ); + }); + + test('a video-search error is shown inline and does not hide photo results', async () => { + (searchImagesByTag as jest.Mock).mockResolvedValue({ + success: true, + data: [ + { + id: '1', + path: '/img1.jpg', + thumbnailPath: '/thumb1.jpg', + tags: ['cat'], + }, + ], + }); + (searchVideosByTag as jest.Mock).mockRejectedValue( + new Error('Network Error'), + ); + + renderWithQuery('cat'); + + await waitFor( + () => { + // Inline video error, not the full-screen takeover... + expect( + screen.getByText(/Couldn't load video results/i), + ).toBeInTheDocument(); + expect(screen.queryByText(/^Search Failed$/i)).not.toBeInTheDocument(); + // ...and the photos still render. + expect( + screen.queryByText(/Couldn't load photo results/i), + ).not.toBeInTheDocument(); + }, + { timeout: 3000 }, + ); + }); + + test('the Search Failed takeover appears only when both searches fail', async () => { (searchImagesByTag as jest.Mock).mockRejectedValue( new Error('Network Error'), ); + (searchVideosByTag as jest.Mock).mockRejectedValue( + new Error('Network Error'), + ); renderWithQuery('dog'); diff --git a/frontend/src/types/Folder.ts b/frontend/src/types/Folder.ts index a4e7c032e..4ec7024dc 100644 --- a/frontend/src/types/Folder.ts +++ b/frontend/src/types/Folder.ts @@ -6,6 +6,7 @@ export interface FolderDetails { AI_Tagging: boolean; taggingCompleted?: boolean; image_count?: number; + video_count?: number; indexing_status?: 'not_started' | 'in_progress' | 'completed'; } diff --git a/frontend/src/types/FolderStatus.ts b/frontend/src/types/FolderStatus.ts index cf68ad3e0..950f6ef6a 100644 --- a/frontend/src/types/FolderStatus.ts +++ b/frontend/src/types/FolderStatus.ts @@ -6,6 +6,9 @@ export interface FolderTaggingInfo { total_images: number; tagged_images: number; embedded_images: number; + total_videos: number; + tagged_videos: number; + embedded_videos: number; ai_tagging: boolean; } diff --git a/frontend/src/types/Media.ts b/frontend/src/types/Media.ts index 57f8b3a2b..bfc022f97 100644 --- a/frontend/src/types/Media.ts +++ b/frontend/src/types/Media.ts @@ -48,6 +48,12 @@ export interface Video { tags?: string[]; } +export interface ScoredVideo extends Video { + score: number; + /** Timestamp (seconds) of the keyframe that matched best. */ + best_frame_timestamp?: number | null; +} + export interface ImageGalleryProps { mediaItems: Image[]; title?: string; diff --git a/sync-microservice/app/database/folders.py b/sync-microservice/app/database/folders.py index 22429992a..6447445ca 100644 --- a/sync-microservice/app/database/folders.py +++ b/sync-microservice/app/database/folders.py @@ -12,7 +12,12 @@ class FolderTaggingInfo(NamedTuple): - """Represents folder tagging and semantic-embedding information""" + """Represents folder tagging and semantic-embedding information. + + Percentages are over images and videos combined, so a video-only folder + still reports real progress. The raw per-medium counts are exposed so the + frontend can aggregate them however it needs. + """ folder_id: FolderId folder_path: FolderPath @@ -21,6 +26,9 @@ class FolderTaggingInfo(NamedTuple): total_images: int tagged_images: int embedded_images: int + total_videos: int + tagged_videos: int + embedded_videos: int ai_tagging: bool @@ -80,28 +88,57 @@ def db_check_database_connection() -> bool: def db_get_tagging_progress() -> List[FolderTaggingInfo]: """ Calculate tagging and semantic-embedding percentages for all folders. - Each percentage = (processed images / total images) * 100. + Each percentage = (processed media / total media) * 100, counting images + and videos together. Returns: - List of FolderTaggingInfo with percentages, raw counts, and each - folder's AI_Tagging flag + List of FolderTaggingInfo with percentages, raw per-medium counts, and + each folder's AI_Tagging flag """ conn = sqlite3.connect(DATABASE_PATH) cursor = conn.cursor() try: + # Pre-aggregate each media table by folder before joining, so a folder + # with M images and N videos never fans out to M*N intermediate rows. + # A video is "embedded" once tagged with no unembedded frames left -- a + # zero-frame (undecodable) video counts as embedded so it never stalls + # the bar. cursor.execute( """ SELECT f.folder_id, f.folder_path, f.AI_Tagging, - COUNT(i.id) as total_images, - COUNT(CASE WHEN i.isTagged = 1 THEN 1 END) as tagged_images, - COUNT(CASE WHEN i.isEmbedded = 1 THEN 1 END) as embedded_images + COALESCE(img.total_images, 0) as total_images, + COALESCE(img.tagged_images, 0) as tagged_images, + COALESCE(img.embedded_images, 0) as embedded_images, + COALESCE(vid.total_videos, 0) as total_videos, + COALESCE(vid.tagged_videos, 0) as tagged_videos, + COALESCE(vid.embedded_videos, 0) as embedded_videos FROM folders f - LEFT JOIN images i ON f.folder_id = i.folder_id - GROUP BY f.folder_id, f.folder_path, f.AI_Tagging + LEFT JOIN ( + SELECT folder_id, + COUNT(*) as total_images, + SUM(CASE WHEN isTagged = 1 THEN 1 ELSE 0 END) + as tagged_images, + SUM(CASE WHEN isEmbedded = 1 THEN 1 ELSE 0 END) + as embedded_images + FROM images + GROUP BY folder_id + ) img ON img.folder_id = f.folder_id + LEFT JOIN ( + SELECT v2.folder_id, + COUNT(*) as total_videos, + SUM(CASE WHEN v2.isTagged = 1 THEN 1 ELSE 0 END) + as tagged_videos, + SUM(CASE WHEN v2.isTagged = 1 AND NOT EXISTS ( + SELECT 1 FROM video_frames vf + WHERE vf.video_id = v2.id AND vf.isEmbedded = 0 + ) THEN 1 ELSE 0 END) as embedded_videos + FROM videos v2 + GROUP BY v2.folder_id + ) vid ON vid.folder_id = f.folder_id """ ) @@ -115,11 +152,17 @@ def db_get_tagging_progress() -> List[FolderTaggingInfo]: total_images, tagged_images, embedded_images, + total_videos, + tagged_videos, + embedded_videos, ) in results: - # Calculate percentages, handle division by zero - if total_images > 0: - tagging_percentage = (tagged_images / total_images) * 100 - embedding_percentage = (embedded_images / total_images) * 100 + # Percentages are over images and videos together. + total_media = total_images + total_videos + if total_media > 0: + tagging_percentage = (tagged_images + tagged_videos) / total_media * 100 + embedding_percentage = ( + (embedded_images + embedded_videos) / total_media * 100 + ) else: tagging_percentage = 0.0 embedding_percentage = 0.0 @@ -133,6 +176,9 @@ def db_get_tagging_progress() -> List[FolderTaggingInfo]: total_images=total_images, tagged_images=tagged_images, embedded_images=embedded_images, + total_videos=total_videos, + tagged_videos=tagged_videos, + embedded_videos=embedded_videos, ai_tagging=bool(ai_tagging), ) ) diff --git a/sync-microservice/app/routes/folders.py b/sync-microservice/app/routes/folders.py index 115458935..b2285ea2d 100644 --- a/sync-microservice/app/routes/folders.py +++ b/sync-microservice/app/routes/folders.py @@ -36,6 +36,9 @@ def get_folders_tagging_status(): total_images=folder.total_images, tagged_images=folder.tagged_images, embedded_images=folder.embedded_images, + total_videos=folder.total_videos, + tagged_videos=folder.tagged_videos, + embedded_videos=folder.embedded_videos, ai_tagging=folder.ai_tagging, ) for folder in tagging_progress diff --git a/sync-microservice/app/schemas/folders.py b/sync-microservice/app/schemas/folders.py index d3aefdf9a..48ca894be 100644 --- a/sync-microservice/app/schemas/folders.py +++ b/sync-microservice/app/schemas/folders.py @@ -11,13 +11,13 @@ class FolderTaggingInfo(BaseModel): ..., ge=0, le=100, - description="Percentage of images that have been tagged (0-100)", + description="Percentage of images and videos tagged (0-100)", ) embedding_percentage: float = Field( ..., ge=0, le=100, - description="Percentage of images with semantic embeddings (0-100)", + description="Percentage of images and videos with embeddings (0-100)", ) total_images: int = Field( ..., ge=0, description="Total number of images in the folder" @@ -28,6 +28,15 @@ class FolderTaggingInfo(BaseModel): embedded_images: int = Field( ..., ge=0, description="Number of images with semantic embeddings" ) + total_videos: int = Field( + ..., ge=0, description="Total number of videos in the folder" + ) + tagged_videos: int = Field( + ..., ge=0, description="Number of videos that have been tagged" + ) + embedded_videos: int = Field( + ..., ge=0, description="Number of videos whose frames are embedded" + ) ai_tagging: bool = Field( ..., description="Whether AI tagging is enabled for the folder" )