Add TwelveLabs Marengo embedding integration for video vector search

docs/docs/guide/twelvelabs.md

@@ -0,0 +1,75 @@
+---
+seo_title: "TwelveLabs Marengo embeddings for Deep Lake | Multimodal video vector search"
+description: "Build a multimodal video vector store in Deep Lake using TwelveLabs Marengo 512-dim embeddings for text and video."
+---
+
+# Video Vector Search with TwelveLabs Marengo
+
+[TwelveLabs](https://twelvelabs.io) Marengo is a multimodal embedding model that maps
+text, images, audio, and **video** into the same 512-dimensional vector space. Because
+Deep Lake stores raw video alongside embeddings and runs vector search client-side, the
+two pair naturally: embed your videos at ingestion time, embed natural-language queries
+at search time, and run cosine-similarity search over a single `Embedding` column.
+
+This integration is fully opt-in and does not change any Deep Lake defaults.
+
+## Install
+
+```bash
+pip install deeplake twelvelabs
+```
+
+Set your TwelveLabs API key (grab a free one at [twelvelabs.io](https://twelvelabs.io) —
+there's a generous free tier):
+
+```python
+import os
+os.environ["TWELVELABS_API_KEY"] = "<your_api_key>"
+```
+
+## Create a dataset and embed videos
+
+`MarengoEmbedder.dim` is `512`, so it plugs straight into `types.Embedding`. The
+`embed_videos` method accepts public video URLs and returns one vector per video.
+
+```python
+import deeplake
+from deeplake import types
+from deeplake.integrations.twelvelabs import MarengoEmbedder
+
+embedder = MarengoEmbedder()  # reads TWELVELABS_API_KEY from the environment
+
+ds = deeplake.create("mem://videos")
+ds.add_column("url", types.Text())
+ds.add_column("embedding", types.Embedding(embedder.dim))
+
+video_urls = [
+    "https://example.com/cooking.mp4",
+    "https://example.com/traffic.mp4",
+]
+ds.append({
+    "url": video_urls,
+    "embedding": embedder.embed_videos(video_urls),
+})
+ds.commit()
+```
+
+## Search with a text query
+
+Because Marengo embeds text into the same space as video, you can search your videos
+with natural language. `embed_text` follows the same calling convention as the
+`embedding_function` used throughout the Deep Lake RAG guides (string or list of strings
+in, list of vectors out).
+
+```python
+query_vector = embedder.embed_text("a chef plating a dish in a busy kitchen")[0]
+query_str = ",".join(str(c) for c in query_vector)
+
+view = ds.query(
+    f"SELECT *, COSINE_SIMILARITY(embedding, ARRAY[{query_str}]) AS score "
+    f"ORDER BY score DESC LIMIT 5"
+)
+```
+
+`embed_text` also accepts a list of strings and returns the vectors in order, so it can
+be dropped into existing batched ingestion pipelines.

python/deeplake/integrations/twelvelabs/__init__.py

@@ -0,0 +1,5 @@
+from .marengo import (
+    MarengoEmbedder,
+    MARENGO_EMBEDDING_DIM,
+    DEFAULT_MARENGO_MODEL,
+)

python/deeplake/integrations/twelvelabs/marengo.py

@@ -0,0 +1,144 @@
+import os
+
+from typing import List, Optional, Sequence, Union
+
+# TwelveLabs Marengo produces 512-dimensional multimodal embeddings.
+MARENGO_EMBEDDING_DIM = 512
+
+# Default Marengo model name. See https://docs.twelvelabs.io for available models.
+DEFAULT_MARENGO_MODEL = "marengo3.0"
+
+
+class MarengoEmbedder:
+    """Generates multimodal embeddings with TwelveLabs Marengo for Deep Lake datasets.
+
+    Marengo maps text, images, and audio into the **same** 512-dimensional vector
+    space, which makes it a good fit for building multimodal vector stores in Deep
+    Lake: embed your media at ingestion time and embed natural-language queries at
+    search time, then run cosine-similarity search against a single
+    :func:`deeplake.types.Embedding` column.
+
+    This integration is fully opt-in. It does not change any Deep Lake defaults and
+    requires the optional ``twelvelabs`` package (``pip install twelvelabs``) plus a
+    TwelveLabs API key. You can grab a free API key at https://twelvelabs.io.
+
+    Args:
+        api_key (str, optional): TwelveLabs API key. If omitted, the ``TWELVELABS_API_KEY``
+            environment variable is used.
+        model_name (str, optional): Marengo model to use. Defaults to ``"marengo3.0"``.
+
+    Raises:
+        ValueError: If no API key is provided and ``TWELVELABS_API_KEY`` is not set.
+
+    Example:
+        >>> import deeplake
+        >>> from deeplake import types
+        >>> from deeplake.integrations.twelvelabs import MarengoEmbedder
+        >>>
+        >>> embedder = MarengoEmbedder(api_key="<your_api_key>")
+        >>>
+        >>> ds = deeplake.create("mem://videos")
+        >>> ds.add_column("embedding", types.Embedding(embedder.dim))
+        >>> ds.add_column("url", types.Text())
+        >>>
+        >>> # Embed videos at ingestion time.
+        >>> video_urls = ["https://example.com/clip.mp4"]
+        >>> ds.append({
+        ...     "url": video_urls,
+        ...     "embedding": embedder.embed_videos(video_urls),
+        ... })
+        >>> ds.commit()
+        >>>
+        >>> # Embed a text query at search time and run vector search.
+        >>> query = ",".join(str(c) for c in embedder.embed_text("a chef plating a dish"))
+        >>> view = ds.query(
+        ...     f"SELECT * ORDER BY COSINE_SIMILARITY(embedding, ARRAY[{query}]) DESC LIMIT 5"
+        ... )
+
+    Notes:
+        - All embeddings are 512-dimensional vectors in a shared multimodal space,
+          suitable for cosine-similarity search.
+        - ``embed_text`` follows the same calling convention as the
+          ``embedding_function`` used throughout the Deep Lake RAG guides
+          (string or list of strings in, list of vectors out), so it can be dropped
+          into existing pipelines.
+    """
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        model_name: str = DEFAULT_MARENGO_MODEL,
+    ):
+        api_key = api_key or os.environ.get("TWELVELABS_API_KEY")
+        if not api_key:
+            raise ValueError(
+                "A TwelveLabs API key is required. Pass api_key=... or set the "
+                "TWELVELABS_API_KEY environment variable. Get a free key at "
+                "https://twelvelabs.io."
+            )
+
+        from twelvelabs import TwelveLabs  # type: ignore
+
+        self._client = TwelveLabs(api_key=api_key)
+        self.model_name = model_name
+
+    @property
+    def dim(self) -> int:
+        """Embedding dimensionality (512), for use with ``types.Embedding(embedder.dim)``."""
+        return MARENGO_EMBEDDING_DIM
+
+    @staticmethod
+    def _first_segment_vector(embedding) -> List[float]:
+        if embedding is None or not embedding.segments:
+            raise ValueError(
+                "TwelveLabs returned no embedding segments. "
+                f"error_message={getattr(embedding, 'error_message', None)}"
+            )
+        return list(embedding.segments[0].float_)
+
+    def embed_text(self, texts: Union[str, Sequence[str]]) -> List[List[float]]:
+        """Embed one or more text strings into 512-dimensional vectors.
+
+        Args:
+            texts (str | Sequence[str]): A single string or a sequence of strings.
+
+        Returns:
+            List[List[float]]: One 512-dimensional vector per input string.
+        """
+        if isinstance(texts, str):
+            texts = [texts]
+
+        vectors = []
+        for text in texts:
+            response = self._client.embed.create(model_name=self.model_name, text=text)
+            vectors.append(self._first_segment_vector(response.text_embedding))
+        return vectors
+
+    def embed_videos(self, video_urls: Union[str, Sequence[str]]) -> List[List[float]]:
+        """Embed one or more videos (by public URL) into 512-dimensional vectors.
+
+        Marengo segments a video and returns one embedding per segment; this method
+        returns the first segment's vector for each video so the output lines up
+        one-to-one with the input, ready to append to an ``Embedding`` column. For
+        per-segment access, call the TwelveLabs SDK directly.
+
+        Args:
+            video_urls (str | Sequence[str]): A single video URL or a sequence of URLs.
+
+        Returns:
+            List[List[float]]: One 512-dimensional vector per input video.
+        """
+        if isinstance(video_urls, str):
+            video_urls = [video_urls]
+
+        vectors = []
+        for url in video_urls:
+            task = self._client.embed.tasks.create(
+                model_name=self.model_name, video_url=url
+            )
+            self._client.embed.tasks.wait_for_done(task_id=task.id)
+            result = self._client.embed.tasks.retrieve(
+                task_id=task.id, embedding_option=["visual-text"]
+            )
+            vectors.append(self._first_segment_vector(result.video_embedding))
+        return vectors

python/deeplake/integrations/twelvelabs/test_marengo.py

@@ -0,0 +1,49 @@
+import os
+
+import pytest
+
+from deeplake.integrations.twelvelabs import (
+    MarengoEmbedder,
+    MARENGO_EMBEDDING_DIM,
+)
+
+requires_api_key = pytest.mark.skipif(
+    not os.environ.get("TWELVELABS_API_KEY"),
+    reason="TWELVELABS_API_KEY not set",
+)
+
+
+def test_missing_api_key_raises(monkeypatch):
+    # No-network: constructor must fail clearly when no key is available.
+    monkeypatch.delenv("TWELVELABS_API_KEY", raising=False)
+    with pytest.raises(ValueError, match="TwelveLabs API key"):
+        MarengoEmbedder()
+
+
+def test_first_segment_vector_empty_raises():
+    # No-network: empty / errored embedding responses surface a clear error.
+    class _Empty:
+        segments = []
+        error_message = "boom"
+
+    with pytest.raises(ValueError, match="no embedding segments"):
+        MarengoEmbedder._first_segment_vector(_Empty())
+
+    with pytest.raises(ValueError, match="no embedding segments"):
+        MarengoEmbedder._first_segment_vector(None)
+
+
+@requires_api_key
+def test_embed_text_returns_512_dim_vectors():
+    embedder = MarengoEmbedder()
+    assert embedder.dim == MARENGO_EMBEDDING_DIM == 512
+
+    vectors = embedder.embed_text("a chef plating a dish in a busy kitchen")
+    assert len(vectors) == 1
+    assert len(vectors[0]) == 512
+    assert all(isinstance(x, float) for x in vectors[0])
+
+    # Accepts a list and preserves order / count.
+    batch = embedder.embed_text(["a dog running", "city traffic at night"])
+    assert len(batch) == 2
+    assert all(len(v) == 512 for v in batch)