Upgrade & fix GenTL multi-camera mode

dlclivegui/cameras/backends/utils/gentl_discovery.py

@@ -5,7 +5,9 @@
 from __future__ import annotations
 
 import glob
+import logging
 import os
+import threading
 from collections.abc import Iterable, Sequence
 from dataclasses import dataclass, field
 from enum import Enum, auto
@@ -18,6 +20,125 @@ class GenTLDiscoveryPolicy(Enum):
     RAISE_IF_MULTIPLE = auto()  # if > N candidates, raise an error to avoid ambiguity (forces explicit config)
 
 
+try:  # pragma: no cover - optional dependency
+    from harvesters.core import Harvester  # type: ignore
+except Exception:  # pragma: no cover - optional dependency
+    Harvester = None  # type: ignore
+
+logger = logging.getLogger(__name__)
+
+
+class SharedHarvesterEntry:
+    """
+    A shared Harvester instance keyed by a canonical tuple of CTI files.
+    """
+
+    def __init__(self, cti_files: list[str]):
+        if Harvester is None:  # pragma: no cover
+            raise RuntimeError(
+                "The 'harvesters' package is required for the GenTL backend. Install it via 'pip install harvesters'."
+            )
+
+        self.lock = threading.RLock()
+        self.key = tuple(sorted(_normalize_path(p, casefold_windows=True) for p in cti_files))
+        self.refcount = 0
+        self.harvester = Harvester()
+        self.loaded_files: list[str] = []
+        self.failed_files: dict[str, str] = {}
+
+        for cti in self.key:
+            try:
+                self.harvester.add_file(cti)
+                self.loaded_files.append(cti)
+            except Exception as e:
+                logger.warning(f"Failed to load CTI file: {cti}. Skipping.")
+                self.failed_files[cti] = str(e)
+
+        if not self.loaded_files:
+            e = RuntimeError("No GenTL producer (.cti) could be loaded by shared Harvester.")
+            self._raise_and_reset_harvester(e)
+
+        # Initial device enumeration.
+        try:
+            self.harvester.update()
+        except Exception as e:
+            self._raise_and_reset_harvester(e)
+
+    def _raise_and_reset_harvester(self, exc: Exception) -> None:
+        exc.loaded_files = self.loaded_files[:]
+        exc.failed_files = dict(self.failed_files)
+        try:
+            self.harvester.reset()
+        except Exception:
+            pass
+        raise exc
+
+
+class SharedHarvesterPool:
+    """
+    Process-local pool of shared Harvester instances.
+
+    Keyed by the canonicalized CTI file set.
+    """
+
+    _lock = threading.RLock()
+    _entries: dict[tuple[str, ...], SharedHarvesterEntry] = {}
+
+    @classmethod
+    def acquire(cls, cti_files: list[str]) -> SharedHarvesterEntry:
+        key = tuple(sorted(_normalize_path(p, casefold_windows=True) for p in cti_files))
+        with cls._lock:
+            entry = cls._entries.get(key)
+            if entry is None:
+                entry = SharedHarvesterEntry(list(key))
+                cls._entries[key] = entry
+            entry.refcount += 1
+            return entry
+
+    @classmethod
+    def release(cls, entry: SharedHarvesterEntry | None) -> None:
+        if entry is None:
+            return
+
+        with cls._lock:
+            current = cls._entries.get(entry.key)
+            if current is None:
+                # Already released/reset.
+                return
+
+            current.refcount -= 1
+            if current.refcount > 0:
+                return
+
+            try:
+                with current.lock:
+                    try:
+                        current.harvester.reset()
+                    except Exception:
+                        pass
+            finally:
+                cls._entries.pop(entry.key, None)
+
+    @classmethod
+    def refresh(cls, entry: SharedHarvesterEntry | None) -> None:
+        """
+        Optional helper when callers want to re-enumerate the device list
+        on an already-shared Harvester instance.
+        """
+        if entry is None:
+            return
+        with entry.lock:
+            entry.harvester.update()
+
+    @classmethod
+    def get_refcount(cls, entry: SharedHarvesterEntry | None) -> int:
+        if entry is None:
+            return 0
+        with cls._lock:
+            current = cls._entries.get(entry.key)
+            return int(current.refcount) if current is not None else 0
+
+
 @dataclass
 class CTIDiscoveryDiagnostics:
     explicit_files: list[str] = field(default_factory=list)
@@ -81,18 +202,18 @@ def _expand_user_and_env(value: str) -> str:
     return s
 
 
-def _normalize_path(p: str) -> str:
-    """
-    Normalize a filesystem path in a cross-platform way:
-    - expands ~ and environment variables
-    - resolves to absolute where possible (without requiring existence)
-    """
+def _normalize_path(p: str, *, casefold_windows: bool = False) -> str:
     expanded = _expand_user_and_env(p)
     pp = Path(expanded)
     try:
-        return str(pp.resolve(strict=False))
+        out = str(pp.resolve(strict=False))
     except Exception:
-        return str(pp.absolute())
+        out = str(pp.absolute())
+
+    if casefold_windows:
+        out = os.path.normcase(out)
+
+    return out
 
 
 def _iter_cti_files_in_dir(directory: str, recursive: bool = False) -> Iterable[str]:

dlclivegui/main.py

@@ -3,6 +3,7 @@
 
 import argparse
 import logging
+import os
 import signal
 import sys
 
@@ -54,6 +55,32 @@ def _sigint_handler(_signum, _frame) -> None:
     app._sig_timer = sig_timer  # Store on app to keep it alive and allow cleanup on exit
 
 
+def configure_logging(debug: bool = False) -> None:
+    """Configure local application logging."""
+    env_debug = os.environ.get("DLCLIVEGUI_DEBUG_LOGGING", "").strip().lower() in (
+        "1",
+        "true",
+        "yes",
+        "on",
+        "debug",
+    )
+
+    enabled = bool(debug or env_debug)
+    level = logging.DEBUG if enabled else logging.INFO
+
+    logging.basicConfig(
+        level=level,
+        format="%(asctime)s.%(msecs)03d %(levelname)-8s [%(threadName)s] %(name)s:%(lineno)d - %(message)s",
+        datefmt="%H:%M:%S",
+        force=True,
+    )
+
+    logging.getLogger("dlclivegui").setLevel(level)
+
+    if enabled:
+        logging.debug("Debug logging enabled.")
+
+
 def parse_args(argv=None):
     if argv is None:
         argv = sys.argv[1:]
@@ -77,12 +104,13 @@ def parse_args(argv=None):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     parser.add_argument("--no-art", action="store_true", help="Disable ASCII art in help and when launching.")
+    parser.add_argument("--debug-log", action="store_true", help="Enable debug logging.")
     return parser.parse_known_args(argv)
 
 
 def main() -> None:
     args, _unknown = parse_args()
-
+    configure_logging(debug=args.debug_log)
     logging.info("Starting DeepLabCut-Live GUI...")
 
     # If you want a startup banner, PRINT it (not log), and only in TTY contexts.

dlclivegui/services/multi_camera_controller.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import copy
 import logging
 import time
 from dataclasses import dataclass
@@ -14,6 +15,7 @@
 
 from dlclivegui.cameras import CameraFactory
 from dlclivegui.cameras.base import CameraBackend
+from dlclivegui.cameras.factory import camera_identity_key
 
 # from dlclivegui.config import CameraSettings
 from dlclivegui.config import CameraSettings
@@ -29,6 +31,7 @@ class MultiFrameData:
     timestamps: dict[str, float]  # camera_id -> timestamp
     source_camera_id: str = ""  # ID of camera that triggered this emission
     tiled_frame: np.ndarray | None = None  # Combined tiled frame (deprecated, done in GUI)
+    display_ids: dict[str, str] = None  # camera_id -> display_id (for labeling)
 
 
 class SingleCameraWorker(QObject):
@@ -42,7 +45,7 @@ class SingleCameraWorker(QObject):
     def __init__(self, camera_id: str, settings: CameraSettings):
         super().__init__()
         self._camera_id = camera_id
-        self._settings = settings
+        self._settings = copy.deepcopy(settings)
         self._stop_event = Event()
         self._backend: CameraBackend | None = None
         self._max_consecutive_errors = 5
@@ -53,7 +56,24 @@ def run(self) -> None:
         self._stop_event.clear()
 
         try:
+            LOGGER.debug(
+                "[Worker %s] before create: backend=%s index=%s properties=%s",
+                self._camera_id,
+                self._settings.backend,
+                self._settings.index,
+                self._settings.properties,
+            )
+
             self._backend = CameraFactory.create(self._settings)
+
+            LOGGER.debug(
+                "[Worker %s] after create: backend=%s index=%s properties=%s",
+                self._camera_id,
+                self._backend.settings.backend,
+                self._backend.settings.index,
+                self._backend.settings.properties,
+            )
+
             self._backend.open()
         except Exception as exc:
             LOGGER.exception(f"Failed to initialize camera {self._camera_id}", exc_info=exc)
@@ -102,11 +122,27 @@ def stop(self) -> None:
         self._stop_event.set()
 
 
-def get_camera_id(settings: CameraSettings) -> str:
-    """Generate a unique camera ID from settings."""
+def get_display_id(settings: CameraSettings) -> str:
     return f"{settings.backend}:{settings.index}"
 
 
+def get_camera_id(settings: CameraSettings) -> str:
+    """Generate a unique camera ID from stable backend identity."""
+    backend = (settings.backend or "").lower()
+    props = settings.properties if isinstance(settings.properties, dict) else {}
+    ns = props.get(backend, {}) if isinstance(props.get(backend), dict) else {}
+
+    device_id = ns.get("device_id")
+    if device_id:
+        return f"{backend}:{device_id}"
+
+    serial = ns.get("serial_number") or ns.get("device_serial_number") or ns.get("serial")
+    if serial:
+        return f"{backend}:serial:{serial}"
+
+    return f"{backend}:index:{int(settings.index)}"
+
+
 class MultiCameraController(QObject):
     """Controller for managing multiple cameras simultaneously."""
 
@@ -131,6 +167,7 @@ def __init__(self):
         self._frame_lock = Lock()
         self._running = False
         self._started_cameras: set = set()
+        self._display_ids: dict[str, str] = {}  # camera_id -> display_id (for labeling)
         self._failed_cameras: dict[str, str] = {}  # camera_id -> error message
         self._expected_cameras: int = 0  # Number of cameras we're trying to start
 
@@ -153,25 +190,58 @@ def start(self, camera_settings: list[CameraSettings]) -> None:
             LOGGER.warning("No active cameras to start")
             return
 
+        # Check for dupes
+        seen = {}
+        for s in active_settings:
+            camera_id = get_camera_id(s)
+            try:
+                key = camera_identity_key(s)
+            except Exception:
+                LOGGER.exception(
+                    "Failed to compute camera identity key for %s; falling back to camera_id",
+                    camera_id,
+                )
+                key = camera_id
+
+            if key in seen:
+                self.initialization_failed.emit(
+                    [
+                        (
+                            camera_id,
+                            f"Duplicate camera configuration. Conflicts with {seen[key]}",
+                        )
+                    ]
+                )
+                return
+
+            seen[key] = camera_id
+
         self._running = True
         self._frames.clear()
         self._timestamps.clear()
         self._started_cameras.clear()
         self._failed_cameras.clear()
+        self._display_ids.clear()
         self._expected_cameras = len(active_settings)
 
         for settings in active_settings:
             self._start_camera(settings)
 
     def _start_camera(self, settings: CameraSettings) -> None:
         """Start a single camera."""
-        cam_id = get_camera_id(settings)
+        settings_copy = copy.deepcopy(settings)
+        cam_id = get_camera_id(settings_copy)
+        display_id = get_display_id(settings_copy)
+
         if cam_id in self._workers:
             LOGGER.warning(f"Camera {cam_id} already has a worker")
             return
 
+        LOGGER.info(f"[MultiCameraController] Starting {cam_id} with settings: {settings_copy}")
+
         # Normalize and store the dataclass once
-        self._settings[cam_id] = settings
+        self._settings[cam_id] = settings_copy
+        self._display_ids[cam_id] = display_id
         dc = self._settings[cam_id]
         worker = SingleCameraWorker(cam_id, dc)
         thread = QThread()
@@ -211,6 +281,7 @@ def stop(self, wait: bool = True) -> None:
         self._settings.clear()
         self._started_cameras.clear()
         self._failed_cameras.clear()
+        self._display_ids.clear()
         self._expected_cameras = 0
         self.all_stopped.emit()
 
@@ -238,6 +309,7 @@ def _on_frame_captured(self, camera_id: str, frame: np.ndarray, timestamp: float
                     timestamps=dict(self._timestamps),
                     source_camera_id=camera_id,  # Track which camera triggered this
                     tiled_frame=None,
+                    display_ids=dict(self._display_ids),
                 )
                 self.frame_ready.emit(frame_data)
 
@@ -425,6 +497,7 @@ def _on_camera_stopped(self, camera_id: str) -> None:
         # Check if this camera never started (initialization failure)
         was_started = camera_id in self._started_cameras
         self._started_cameras.discard(camera_id)
+        self._display_ids.pop(camera_id, None)
         self.camera_stopped.emit(camera_id)
         LOGGER.info(f"Camera {camera_id} stopped (was_started={was_started})")