Add custom vectorizer class (#156)

This PR introduces a new CustomTextVectorizer class that allows for
users to wrap their own embedding functions and methods to be compatible
with RedisVL.

Author: justin-cechmanek

docs/user_guide/vectorizers_04.ipynb

@@ -13,6 +13,7 @@
     "3. Vertex AI\n",
     "4. Cohere\n",
     "5. Mistral AI\n",
+    "6. Bringing your own vectorizer\n",
     "\n",
     "Before running this notebook, be sure to\n",
     "1. Have installed ``redisvl`` and have that environment active for this notebook.\n",
@@ -90,16 +91,16 @@
     {
      "data": {
       "text/plain": [
-       "[-0.001025049015879631,\n",
-       " -0.0030993607360869646,\n",
-       " 0.0024536605924367905,\n",
-       " -0.004484387580305338,\n",
-       " -0.010331203229725361,\n",
-       " 0.012700922787189484,\n",
-       " -0.005368996877223253,\n",
-       " -0.0029411641880869865,\n",
-       " -0.0070833307690918446,\n",
-       " -0.03386051580309868]"
+       "[-0.0010508307022973895,\n",
+       " -0.0031670420430600643,\n",
+       " 0.0023781107738614082,\n",
+       " -0.004539588466286659,\n",
+       " -0.010320774279534817,\n",
+       " 0.012868634425103664,\n",
+       " -0.0054513863287866116,\n",
+       " -0.002984359161928296,\n",
+       " -0.0072814482264220715,\n",
+       " -0.033704183995723724]"
       ]
      },
      "execution_count": 3,
@@ -129,16 +130,16 @@
     {
      "data": {
       "text/plain": [
-       "[-0.01747742109000683,\n",
-       " -5.228330701356754e-05,\n",
-       " 0.0013870716793462634,\n",
-       " -0.025637786835432053,\n",
-       " -0.01985435001552105,\n",
-       " 0.016117358580231667,\n",
-       " -0.0037306349258869886,\n",
-       " 0.0008945261361077428,\n",
-       " 0.006577865686267614,\n",
-       " -0.025091219693422318]"
+       "[-0.01749197021126747,\n",
+       " -5.238811718299985e-05,\n",
+       " 0.0013331907102838159,\n",
+       " -0.025576923042535782,\n",
+       " -0.019907286390662193,\n",
+       " 0.016106342896819115,\n",
+       " -0.003756451653316617,\n",
+       " 0.0009971122490242124,\n",
+       " 0.006661186460405588,\n",
+       " -0.024954024702310562]"
       ]
      },
      "execution_count": 4,
@@ -505,9 +506,10 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Mistral AI\n",
+    "### Mistral AI\n",
     "\n",
-    "[Mistral](https://console.mistral.ai/) offers LLM and embedding APIs you to implement into your product. The `MistralAITextVectorizer` makes it simple to use RedisVL with their embeddings model. You will need to install `mistralai`.\n",
+    "[Mistral](https://console.mistral.ai/) offers LLM and embedding APIs for you to implement into your product. The `MistralAITextVectorizer` makes it simple to use RedisVL with their embeddings model.\n",
+    "You will need to install `mistralai`.\n",
     "\n",
     "```bash\n",
     "pip install mistralai\n",
@@ -516,14 +518,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "vector dimensions: 1024\n",
+      "Vector dimensions:  1024\n",
       "[-0.02801513671875, 0.02532958984375, 0.04278564453125, 0.0185699462890625, 0.041015625, 0.006053924560546875, 0.03607177734375, -0.0030155181884765625, 0.0033893585205078125, -0.01390838623046875]\n"
      ]
     }
@@ -533,12 +535,91 @@
     "\n",
     "mistral = MistralAITextVectorizer()\n",
     "\n",
-    "# embed a sentence using their asyncronous method\n",
+    "# mebed a sentence using their asyncronous method\n",
     "test = await mistral.aembed(\"This is a test sentence.\")\n",
-    "print(\"vector dimensions:\", len(test))\n",
+    "print(\"Vector dimensions: \", len(test))\n",
     "print(test[:10])"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Custom Vectorizers\n",
+    "\n",
+    "RedisVL supports the use of other vectorizers and provides a class to enable compatibility with any function that generates a vector or vectors from string data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[0.101, 0.101, 0.101, 0.101, 0.101, 0.101, 0.101, 0.101, 0.101, 0.101]"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from redisvl.utils.vectorize import CustomTextVectorizer\n",
+    "\n",
+    "def generate_embeddings(text_input):\n",
+    "    return [0.101] * 768\n",
+    "\n",
+    "custom_vectorizer = CustomTextVectorizer(generate_embeddings)\n",
+    "\n",
+    "custom_vectorizer.embed(\"This is a test sentence.\")[:10]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "This enables the use of custom vectorizers with other RedisVL components"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "11:04:14 redisvl.index.index INFO   Index already exists, not overwriting.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "[{'id': 'llmcache:78bd2446a37a0c6ab62652af9b7e53845145c4471ea83ff9fb4280a528d36bbb',\n",
+       "  'vector_distance': '6.13927841187e-06',\n",
+       "  'prompt': 'this is a test prompt',\n",
+       "  'response': 'this is a test response',\n",
+       "  'prompt_vector': '\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17=\\x17='}]"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from redisvl.extensions.llmcache import SemanticCache\n",
+    "\n",
+    "cache = SemanticCache(vectorizer=custom_vectorizer)\n",
+    "\n",
+    "cache.store(\"this is a test prompt\", \"this is a test response\")\n",
+    "cache.check(\"this is also a test prompt\")"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

redisvl/utils/vectorize/__init__.py

@@ -1,6 +1,7 @@
 from redisvl.utils.vectorize.base import BaseVectorizer
 from redisvl.utils.vectorize.text.azureopenai import AzureOpenAITextVectorizer
 from redisvl.utils.vectorize.text.cohere import CohereTextVectorizer
+from redisvl.utils.vectorize.text.custom import CustomTextVectorizer
 from redisvl.utils.vectorize.text.huggingface import HFTextVectorizer
 from redisvl.utils.vectorize.text.mistral import MistralAITextVectorizer
 from redisvl.utils.vectorize.text.openai import OpenAITextVectorizer
@@ -14,4 +15,5 @@
     "VertexAITextVectorizer",
     "AzureOpenAITextVectorizer",
     "MistralAITextVectorizer",
+    "CustomTextVectorizer",
 ]