diff --git a/docs/docs/guide/twelvelabs.md b/docs/docs/guide/twelvelabs.md new file mode 100644 index 0000000000..5eb3442b4c --- /dev/null +++ b/docs/docs/guide/twelvelabs.md @@ -0,0 +1,75 @@ +--- +seo_title: "TwelveLabs Marengo embeddings for Deep Lake | Multimodal video vector search" +description: "Build a multimodal video vector store in Deep Lake using TwelveLabs Marengo 512-dim embeddings for text and video." +--- + +# Video Vector Search with TwelveLabs Marengo + +[TwelveLabs](https://twelvelabs.io) Marengo is a multimodal embedding model that maps +text, images, audio, and **video** into the same 512-dimensional vector space. Because +Deep Lake stores raw video alongside embeddings and runs vector search client-side, the +two pair naturally: embed your videos at ingestion time, embed natural-language queries +at search time, and run cosine-similarity search over a single `Embedding` column. + +This integration is fully opt-in and does not change any Deep Lake defaults. + +## Install + +```bash +pip install deeplake twelvelabs +``` + +Set your TwelveLabs API key (grab a free one at [twelvelabs.io](https://twelvelabs.io) — +there's a generous free tier): + +```python +import os +os.environ["TWELVELABS_API_KEY"] = "" +``` + +## Create a dataset and embed videos + +`MarengoEmbedder.dim` is `512`, so it plugs straight into `types.Embedding`. The +`embed_videos` method accepts public video URLs and returns one vector per video. + +```python +import deeplake +from deeplake import types +from deeplake.integrations.twelvelabs import MarengoEmbedder + +embedder = MarengoEmbedder() # reads TWELVELABS_API_KEY from the environment + +ds = deeplake.create("mem://videos") +ds.add_column("url", types.Text()) +ds.add_column("embedding", types.Embedding(embedder.dim)) + +video_urls = [ + "https://example.com/cooking.mp4", + "https://example.com/traffic.mp4", +] +ds.append({ + "url": video_urls, + "embedding": embedder.embed_videos(video_urls), +}) +ds.commit() +``` + +## Search with a text query + +Because Marengo embeds text into the same space as video, you can search your videos +with natural language. `embed_text` follows the same calling convention as the +`embedding_function` used throughout the Deep Lake RAG guides (string or list of strings +in, list of vectors out). + +```python +query_vector = embedder.embed_text("a chef plating a dish in a busy kitchen")[0] +query_str = ",".join(str(c) for c in query_vector) + +view = ds.query( + f"SELECT *, COSINE_SIMILARITY(embedding, ARRAY[{query_str}]) AS score " + f"ORDER BY score DESC LIMIT 5" +) +``` + +`embed_text` also accepts a list of strings and returns the vectors in order, so it can +be dropped into existing batched ingestion pipelines. diff --git a/python/deeplake/integrations/twelvelabs/__init__.py b/python/deeplake/integrations/twelvelabs/__init__.py new file mode 100644 index 0000000000..78ec3cfd20 --- /dev/null +++ b/python/deeplake/integrations/twelvelabs/__init__.py @@ -0,0 +1,5 @@ +from .marengo import ( + MarengoEmbedder, + MARENGO_EMBEDDING_DIM, + DEFAULT_MARENGO_MODEL, +) diff --git a/python/deeplake/integrations/twelvelabs/marengo.py b/python/deeplake/integrations/twelvelabs/marengo.py new file mode 100644 index 0000000000..bb7539a642 --- /dev/null +++ b/python/deeplake/integrations/twelvelabs/marengo.py @@ -0,0 +1,144 @@ +import os + +from typing import List, Optional, Sequence, Union + +# TwelveLabs Marengo produces 512-dimensional multimodal embeddings. +MARENGO_EMBEDDING_DIM = 512 + +# Default Marengo model name. See https://docs.twelvelabs.io for available models. +DEFAULT_MARENGO_MODEL = "marengo3.0" + + +class MarengoEmbedder: + """Generates multimodal embeddings with TwelveLabs Marengo for Deep Lake datasets. + + Marengo maps text, images, and audio into the **same** 512-dimensional vector + space, which makes it a good fit for building multimodal vector stores in Deep + Lake: embed your media at ingestion time and embed natural-language queries at + search time, then run cosine-similarity search against a single + :func:`deeplake.types.Embedding` column. + + This integration is fully opt-in. It does not change any Deep Lake defaults and + requires the optional ``twelvelabs`` package (``pip install twelvelabs``) plus a + TwelveLabs API key. You can grab a free API key at https://twelvelabs.io. + + Args: + api_key (str, optional): TwelveLabs API key. If omitted, the ``TWELVELABS_API_KEY`` + environment variable is used. + model_name (str, optional): Marengo model to use. Defaults to ``"marengo3.0"``. + + Raises: + ValueError: If no API key is provided and ``TWELVELABS_API_KEY`` is not set. + + Example: + >>> import deeplake + >>> from deeplake import types + >>> from deeplake.integrations.twelvelabs import MarengoEmbedder + >>> + >>> embedder = MarengoEmbedder(api_key="") + >>> + >>> ds = deeplake.create("mem://videos") + >>> ds.add_column("embedding", types.Embedding(embedder.dim)) + >>> ds.add_column("url", types.Text()) + >>> + >>> # Embed videos at ingestion time. + >>> video_urls = ["https://example.com/clip.mp4"] + >>> ds.append({ + ... "url": video_urls, + ... "embedding": embedder.embed_videos(video_urls), + ... }) + >>> ds.commit() + >>> + >>> # Embed a text query at search time and run vector search. + >>> query = ",".join(str(c) for c in embedder.embed_text("a chef plating a dish")) + >>> view = ds.query( + ... f"SELECT * ORDER BY COSINE_SIMILARITY(embedding, ARRAY[{query}]) DESC LIMIT 5" + ... ) + + Notes: + - All embeddings are 512-dimensional vectors in a shared multimodal space, + suitable for cosine-similarity search. + - ``embed_text`` follows the same calling convention as the + ``embedding_function`` used throughout the Deep Lake RAG guides + (string or list of strings in, list of vectors out), so it can be dropped + into existing pipelines. + """ + + def __init__( + self, + api_key: Optional[str] = None, + model_name: str = DEFAULT_MARENGO_MODEL, + ): + api_key = api_key or os.environ.get("TWELVELABS_API_KEY") + if not api_key: + raise ValueError( + "A TwelveLabs API key is required. Pass api_key=... or set the " + "TWELVELABS_API_KEY environment variable. Get a free key at " + "https://twelvelabs.io." + ) + + from twelvelabs import TwelveLabs # type: ignore + + self._client = TwelveLabs(api_key=api_key) + self.model_name = model_name + + @property + def dim(self) -> int: + """Embedding dimensionality (512), for use with ``types.Embedding(embedder.dim)``.""" + return MARENGO_EMBEDDING_DIM + + @staticmethod + def _first_segment_vector(embedding) -> List[float]: + if embedding is None or not embedding.segments: + raise ValueError( + "TwelveLabs returned no embedding segments. " + f"error_message={getattr(embedding, 'error_message', None)}" + ) + return list(embedding.segments[0].float_) + + def embed_text(self, texts: Union[str, Sequence[str]]) -> List[List[float]]: + """Embed one or more text strings into 512-dimensional vectors. + + Args: + texts (str | Sequence[str]): A single string or a sequence of strings. + + Returns: + List[List[float]]: One 512-dimensional vector per input string. + """ + if isinstance(texts, str): + texts = [texts] + + vectors = [] + for text in texts: + response = self._client.embed.create(model_name=self.model_name, text=text) + vectors.append(self._first_segment_vector(response.text_embedding)) + return vectors + + def embed_videos(self, video_urls: Union[str, Sequence[str]]) -> List[List[float]]: + """Embed one or more videos (by public URL) into 512-dimensional vectors. + + Marengo segments a video and returns one embedding per segment; this method + returns the first segment's vector for each video so the output lines up + one-to-one with the input, ready to append to an ``Embedding`` column. For + per-segment access, call the TwelveLabs SDK directly. + + Args: + video_urls (str | Sequence[str]): A single video URL or a sequence of URLs. + + Returns: + List[List[float]]: One 512-dimensional vector per input video. + """ + if isinstance(video_urls, str): + video_urls = [video_urls] + + vectors = [] + for url in video_urls: + task = self._client.embed.tasks.create( + model_name=self.model_name, video_url=url + ) + self._client.embed.tasks.wait_for_done(task_id=task.id) + result = self._client.embed.tasks.retrieve( + task_id=task.id, embedding_option=["visual-text"] + ) + vectors.append(self._first_segment_vector(result.video_embedding)) + return vectors diff --git a/python/deeplake/integrations/twelvelabs/test_marengo.py b/python/deeplake/integrations/twelvelabs/test_marengo.py new file mode 100644 index 0000000000..e50d072f39 --- /dev/null +++ b/python/deeplake/integrations/twelvelabs/test_marengo.py @@ -0,0 +1,49 @@ +import os + +import pytest + +from deeplake.integrations.twelvelabs import ( + MarengoEmbedder, + MARENGO_EMBEDDING_DIM, +) + +requires_api_key = pytest.mark.skipif( + not os.environ.get("TWELVELABS_API_KEY"), + reason="TWELVELABS_API_KEY not set", +) + + +def test_missing_api_key_raises(monkeypatch): + # No-network: constructor must fail clearly when no key is available. + monkeypatch.delenv("TWELVELABS_API_KEY", raising=False) + with pytest.raises(ValueError, match="TwelveLabs API key"): + MarengoEmbedder() + + +def test_first_segment_vector_empty_raises(): + # No-network: empty / errored embedding responses surface a clear error. + class _Empty: + segments = [] + error_message = "boom" + + with pytest.raises(ValueError, match="no embedding segments"): + MarengoEmbedder._first_segment_vector(_Empty()) + + with pytest.raises(ValueError, match="no embedding segments"): + MarengoEmbedder._first_segment_vector(None) + + +@requires_api_key +def test_embed_text_returns_512_dim_vectors(): + embedder = MarengoEmbedder() + assert embedder.dim == MARENGO_EMBEDDING_DIM == 512 + + vectors = embedder.embed_text("a chef plating a dish in a busy kitchen") + assert len(vectors) == 1 + assert len(vectors[0]) == 512 + assert all(isinstance(x, float) for x in vectors[0]) + + # Accepts a list and preserves order / count. + batch = embedder.embed_text(["a dog running", "city traffic at night"]) + assert len(batch) == 2 + assert all(len(v) == 512 for v in batch)