From 7a347484d65bdd1413e50356d59dbd200d6b081a Mon Sep 17 00:00:00 2001 From: Ananya Raval Date: Tue, 12 May 2026 17:33:16 -0400 Subject: [PATCH 1/6] Fix uv command in venv.sh --- venv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/venv.sh b/venv.sh index 4c77c952..72fac55e 100644 --- a/venv.sh +++ b/venv.sh @@ -12,7 +12,7 @@ export UV_CACHE_DIR=/scratch/$(whoami)/uv_cache # To see if the cache directory is set correctly, run the following command # uv config get cache-dir -echo "Cache directory set to: $(uv config get cache-dir)" +echo "Cache directory set to: $(uv cache dir)" # Install dependencies via uv uv sync From f6f32c36381f24b859e80e86d6cddff5ec23921d Mon Sep 17 00:00:00 2001 From: Ananya Raval Date: Tue, 12 May 2026 17:49:09 -0400 Subject: [PATCH 2/6] Update example in README.md --- README.md | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 3aba6eb1..8988eac6 100644 --- a/README.md +++ b/README.md @@ -65,20 +65,22 @@ For more details on the usage of these commands, refer to the [User Guide](https Example: ```python ->>> from vec_inf.api import VecInfClient ->>> client = VecInfClient() ->>> # Assume VEC_INF_ACCOUNT and VEC_INF_WORK_DIR is set ->>> response = client.launch_model("Meta-Llama-3.1-8B-Instruct") ->>> job_id = response.slurm_job_id ->>> status = client.get_status(job_id) ->>> if status.status == ModelStatus.READY: -... print(f"Model is ready at {status.base_url}") ->>> # Alternatively, use wait_until_ready which will either return a StatusResponse or throw a ServerError ->>> try: ->>> status = wait_until_ready(job_id) ->>> except ServerError as e: ->>> print(f"Model launch failed: {e}") ->>> client.shutdown_model(job_id) +from vec_inf.client import VecInfClient +from vec_inf.client.models import ModelStatus + +client = VecInfClient() +# Assume VEC_INF_ACCOUNT and VEC_INF_WORK_DIR is set +response = client.launch_model("Meta-Llama-3.1-8B-Instruct") +job_id = response.slurm_job_id +status = client.get_status(job_id) +if status.status == ModelStatus.READY: + print(f"Model is ready at {status.base_url}") +# Alternatively, use wait_until_ready which will either return a StatusResponse or throw a ServerError +try: + status = wait_until_ready(job_id) + except ServerError as e: + print(f"Model launch failed: {e}") +client.shutdown_model(job_id) ``` For details on the usage of the API, refer to the [API Reference](https://vectorinstitute.github.io/vector-inference/api/) From b2fda0404850a8b791d1b321676521f00b1d8b2f Mon Sep 17 00:00:00 2001 From: Ananya Raval Date: Tue, 12 May 2026 18:00:13 -0400 Subject: [PATCH 3/6] Fix example docstring --- vec_inf/client/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vec_inf/client/api.py b/vec_inf/client/api.py index 3a644152..c10fab98 100644 --- a/vec_inf/client/api.py +++ b/vec_inf/client/api.py @@ -71,7 +71,7 @@ class VecInfClient: Examples -------- - >>> from vec_inf.api import VecInfClient + >>> from vec_inf.client import VecInfClient >>> client = VecInfClient() >>> response = client.launch_model("Meta-Llama-3.1-8B-Instruct") >>> job_id = response.slurm_job_id From d9871f9e70f805b7badfa478364fb9fa1fb23611 Mon Sep 17 00:00:00 2001 From: Ananya Raval Date: Wed, 13 May 2026 18:44:47 -0400 Subject: [PATCH 4/6] Remove try catch block --- README.md | 67 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 8988eac6..f27e6089 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,33 @@ # Vector Inference: Easy inference on Slurm clusters ----------------------------------------------------- +--- -[![PyPI](https://img.shields.io/pypi/v/vec-inf)](https://pypi.org/project/vec-inf) -[![downloads](https://img.shields.io/pypi/dm/vec-inf)](https://pypistats.org/packages/vec-inf) -[![code checks](https://github.com/VectorInstitute/vector-inference/actions/workflows/code_checks.yml/badge.svg)](https://github.com/VectorInstitute/vector-inference/actions/workflows/code_checks.yml) -[![docs](https://github.com/VectorInstitute/vector-inference/actions/workflows/docs.yml/badge.svg)](https://github.com/VectorInstitute/vector-inference/actions/workflows/docs.yml) -[![codecov](https://codecov.io/github/VectorInstitute/vector-inference/branch/main/graph/badge.svg?token=NI88QSIGAC)](https://app.codecov.io/github/VectorInstitute/vector-inference/tree/main) -[![vLLM](https://img.shields.io/badge/vLLM-0.19.0-blue)](https://docs.vllm.ai/en/v0.19.0/) -[![SGLang](https://img.shields.io/badge/SGLang-0.5.9-blue)](https://docs.sglang.io/index.html) -![GitHub License](https://img.shields.io/github/license/VectorInstitute/vector-inference) +[PyPI](https://pypi.org/project/vec-inf) +[downloads](https://pypistats.org/packages/vec-inf) +[code checks](https://github.com/VectorInstitute/vector-inference/actions/workflows/code_checks.yml) +[docs](https://github.com/VectorInstitute/vector-inference/actions/workflows/docs.yml) +[codecov](https://app.codecov.io/github/VectorInstitute/vector-inference/tree/main) +[vLLM](https://docs.vllm.ai/en/v0.19.0/) +[SGLang](https://docs.sglang.io/index.html) +GitHub License This repository provides an easy-to-use solution to run inference servers on [Slurm](https://slurm.schedmd.com/overview.html)-managed computing clusters using open-source inference engines ([vLLM](https://docs.vllm.ai/en/v0.19.0/), [SGLang](https://docs.sglang.io/index.html)). **This package runs natively on the Vector Institute cluster environments**. To adapt to other environments, follow the instructions in [Installation](#installation). ## Installation + If you are using the Vector cluster environment, and you don't need any customization to the inference server environment, run the following to install package: ```bash pip install vec-inf ``` -Otherwise, we recommend using the provided [`vllm.Dockerfile`](vllm.Dockerfile) and [`sglang.Dockerfile`](sglang.Dockerfile) to set up your own environment with the package. The built images are available through [Docker Hub](https://hub.docker.com/orgs/vectorinstitute/repositories) + +Otherwise, we recommend using the provided `[vllm.Dockerfile](vllm.Dockerfile)` and `[sglang.Dockerfile](sglang.Dockerfile)` to set up your own environment with the package. The built images are available through [Docker Hub](https://hub.docker.com/orgs/vectorinstitute/repositories) If you'd like to use `vec-inf` on your own Slurm cluster, you would need to update the configuration files, there are 3 ways to do it: -* Clone the repository and update the `environment.yaml` and the `models.yaml` file in [`vec_inf/config`](vec_inf/config/), then install from source by running `pip install .`. -* The package would try to look for cached configuration files in your environment before using the default configuration. The default cached configuration directory path points to `/model-weights/vec-inf-shared`, you would need to create an `environment.yaml` and a `models.yaml` following the format of these files in [`vec_inf/config`](vec_inf/config/). -* [OPTIONAL] The package could also look for an enviroment variable `VEC_INF_CONFIG_DIR`. You can put your `environment.yaml` and `models.yaml` in a directory of your choice and set the enviroment variable `VEC_INF_CONFIG_DIR` to point to that location. + +- Clone the repository and update the `environment.yaml` and the `models.yaml` file in `[vec_inf/config](vec_inf/config/)`, then install from source by running `pip install .`. +- The package would try to look for cached configuration files in your environment before using the default configuration. The default cached configuration directory path points to `/model-weights/vec-inf-shared`, you would need to create an `environment.yaml` and a `models.yaml` following the format of these files in `[vec_inf/config](vec_inf/config/)`. +- [OPTIONAL] The package could also look for an enviroment variable `VEC_INF_CONFIG_DIR`. You can put your `environment.yaml` and `models.yaml` in a directory of your choice and set the enviroment variable `VEC_INF_CONFIG_DIR` to point to that location. ## Usage @@ -39,24 +42,26 @@ We will use the Llama 3.1 model as example, to launch an OpenAI compatible infer ```bash vec-inf launch Meta-Llama-3.1-8B-Instruct ``` + You should see an output like the following: -launch_image + **NOTE**: You can set the required fields in the environment configuration (`environment.yaml`), it's a mapping between required arguments and their corresponding environment variables. On the Vector **Killarney** Cluster environment, the required fields are: - * `--account`, `-A`: The Slurm account, this argument can be set to default by setting environment variable `VEC_INF_ACCOUNT`. - * `--work-dir`, `-D`: A working directory other than your home directory, this argument can be set to default by seeting environment variable `VEC_INF_WORK_DIR`. -Models that are already supported by `vec-inf` would be launched using the cached configuration (set in [slurm_vars.py](vec_inf/client/slurm_vars.py)) or [default configuration](vec_inf/config/models.yaml). You can override these values by providing additional parameters. Use `vec-inf launch --help` to see the full list of parameters that can be overriden. You can also launch your own custom model as long as the model architecture is supported by the underlying inference engine. For detailed instructions on how to customize your model launch, check out the [`launch` command section in User Guide](https://vectorinstitute.github.io/vector-inference/latest/user_guide/#launch-command). During the launch process, relevant log files and scripts will be written to a log directory (default to `.vec-inf-logs` in your home directory), and a cache directory (`.vec-inf-cache`) will be created in your working directory (defaults to your home directory if not specified or required) for torch compile cache. +- `--account`, `-A`: The Slurm account, this argument can be set to default by setting environment variable `VEC_INF_ACCOUNT`. +- `--work-dir`, `-D`: A working directory other than your home directory, this argument can be set to default by seeting environment variable `VEC_INF_WORK_DIR`. + +Models that are already supported by `vec-inf` would be launched using the cached configuration (set in [slurm_vars.py](vec_inf/client/slurm_vars.py)) or [default configuration](vec_inf/config/models.yaml). You can override these values by providing additional parameters. Use `vec-inf launch --help` to see the full list of parameters that can be overriden. You can also launch your own custom model as long as the model architecture is supported by the underlying inference engine. For detailed instructions on how to customize your model launch, check out the `[launch` command section in User Guide](https://vectorinstitute.github.io/vector-inference/latest/user_guide/#launch-command). During the launch process, relevant log files and scripts will be written to a log directory (default to `.vec-inf-logs` in your home directory), and a cache directory (`.vec-inf-cache`) will be created in your working directory (defaults to your home directory if not specified or required) for torch compile cache. #### Other commands -* `batch-launch`: Launch multiple model inference servers at once, currently ONLY single node models supported, -* `status`: Check the status of all `vec-inf` jobs, or a specific job by providing its job ID. -* `metrics`: Streams performance metrics to the console. -* `shutdown`: Shutdown a model by providing its Slurm job ID. -* `list`: List all available model names, or view the default/cached configuration of a specific model. -* `cleanup`: Remove old log directories, use `--help` to see the supported filters. Use `--dry-run` to preview what would be deleted. +- `batch-launch`: Launch multiple model inference servers at once, currently ONLY single node models supported, +- `status`: Check the status of all `vec-inf` jobs, or a specific job by providing its job ID. +- `metrics`: Streams performance metrics to the console. +- `shutdown`: Shutdown a model by providing its Slurm job ID. +- `list`: List all available model names, or view the default/cached configuration of a specific model. +- `cleanup`: Remove old log directories, use `--help` to see the supported filters. Use `--dry-run` to preview what would be deleted. For more details on the usage of these commands, refer to the [User Guide](https://vectorinstitute.github.io/vector-inference/user_guide/) @@ -73,13 +78,8 @@ client = VecInfClient() response = client.launch_model("Meta-Llama-3.1-8B-Instruct") job_id = response.slurm_job_id status = client.get_status(job_id) -if status.status == ModelStatus.READY: +if status == ModelStatus.READY: print(f"Model is ready at {status.base_url}") -# Alternatively, use wait_until_ready which will either return a StatusResponse or throw a ServerError -try: - status = wait_until_ready(job_id) - except ServerError as e: - print(f"Model launch failed: {e}") client.shutdown_model(job_id) ``` @@ -91,7 +91,7 @@ With every model launch, a Slurm script will be generated dynamically based on t ## Send inference requests -Once the inference server is ready, you can start sending in inference requests. We provide example scripts for sending inference requests in [`examples`](examples) folder. Make sure to update the model server URL and the model weights location in the scripts. For example, you can run `python examples/inference/llm/chat_completions.py`, and you should expect to see an output like the following: +Once the inference server is ready, you can start sending in inference requests. We provide example scripts for sending inference requests in `[examples](examples)` folder. Make sure to update the model server URL and the model weights location in the scripts. For example, you can run `python examples/inference/llm/chat_completions.py`, and you should expect to see an output like the following: ```json { @@ -125,17 +125,23 @@ Once the inference server is ready, you can start sending in inference requests. } ``` + **NOTE**: Certain models don't adhere to OpenAI's chat template, e.g. Mistral family. For these models, you can either change your prompt to follow the model's default chat template or provide your own chat template via `--chat-template: TEMPLATE_PATH`. ## SSH tunnel from your local device + If you want to run inference from your local device, you can open a SSH tunnel to your cluster environment like the following: + ```bash ssh -L 8081:10.1.1.29:8081 username@v.vectorinstitute.ai -N ``` + The example provided above is for the Vector Killarney cluster, change the variables accordingly for your environment. The IP address for the compute nodes on Killarney follow `10.1.1.XX` pattern, where `XX` is the GPU number (`kn029` -> `29` in this example). ## Reference + If you found Vector Inference useful in your research or applications, please cite using the following BibTeX template: + ``` @software{vector_inference, title = {Vector Inference: Efficient LLM inference on Slurm clusters}, @@ -146,3 +152,4 @@ If you found Vector Inference useful in your research or applications, please ci url = {https://github.com/VectorInstitute/vector-inference} } ``` + From ae4c24b654291ab0a8918022d87c329d18ea878b Mon Sep 17 00:00:00 2001 From: Ananya Raval Date: Wed, 13 May 2026 18:54:00 -0400 Subject: [PATCH 5/6] Revert "Remove try catch block" This reverts commit d9871f9e70f805b7badfa478364fb9fa1fb23611. --- README.md | 67 +++++++++++++++++++++++++------------------------------ 1 file changed, 30 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index f27e6089..8988eac6 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,30 @@ # Vector Inference: Easy inference on Slurm clusters ---- +---------------------------------------------------- -[PyPI](https://pypi.org/project/vec-inf) -[downloads](https://pypistats.org/packages/vec-inf) -[code checks](https://github.com/VectorInstitute/vector-inference/actions/workflows/code_checks.yml) -[docs](https://github.com/VectorInstitute/vector-inference/actions/workflows/docs.yml) -[codecov](https://app.codecov.io/github/VectorInstitute/vector-inference/tree/main) -[vLLM](https://docs.vllm.ai/en/v0.19.0/) -[SGLang](https://docs.sglang.io/index.html) -GitHub License +[![PyPI](https://img.shields.io/pypi/v/vec-inf)](https://pypi.org/project/vec-inf) +[![downloads](https://img.shields.io/pypi/dm/vec-inf)](https://pypistats.org/packages/vec-inf) +[![code checks](https://github.com/VectorInstitute/vector-inference/actions/workflows/code_checks.yml/badge.svg)](https://github.com/VectorInstitute/vector-inference/actions/workflows/code_checks.yml) +[![docs](https://github.com/VectorInstitute/vector-inference/actions/workflows/docs.yml/badge.svg)](https://github.com/VectorInstitute/vector-inference/actions/workflows/docs.yml) +[![codecov](https://codecov.io/github/VectorInstitute/vector-inference/branch/main/graph/badge.svg?token=NI88QSIGAC)](https://app.codecov.io/github/VectorInstitute/vector-inference/tree/main) +[![vLLM](https://img.shields.io/badge/vLLM-0.19.0-blue)](https://docs.vllm.ai/en/v0.19.0/) +[![SGLang](https://img.shields.io/badge/SGLang-0.5.9-blue)](https://docs.sglang.io/index.html) +![GitHub License](https://img.shields.io/github/license/VectorInstitute/vector-inference) This repository provides an easy-to-use solution to run inference servers on [Slurm](https://slurm.schedmd.com/overview.html)-managed computing clusters using open-source inference engines ([vLLM](https://docs.vllm.ai/en/v0.19.0/), [SGLang](https://docs.sglang.io/index.html)). **This package runs natively on the Vector Institute cluster environments**. To adapt to other environments, follow the instructions in [Installation](#installation). ## Installation - If you are using the Vector cluster environment, and you don't need any customization to the inference server environment, run the following to install package: ```bash pip install vec-inf ``` - -Otherwise, we recommend using the provided `[vllm.Dockerfile](vllm.Dockerfile)` and `[sglang.Dockerfile](sglang.Dockerfile)` to set up your own environment with the package. The built images are available through [Docker Hub](https://hub.docker.com/orgs/vectorinstitute/repositories) +Otherwise, we recommend using the provided [`vllm.Dockerfile`](vllm.Dockerfile) and [`sglang.Dockerfile`](sglang.Dockerfile) to set up your own environment with the package. The built images are available through [Docker Hub](https://hub.docker.com/orgs/vectorinstitute/repositories) If you'd like to use `vec-inf` on your own Slurm cluster, you would need to update the configuration files, there are 3 ways to do it: - -- Clone the repository and update the `environment.yaml` and the `models.yaml` file in `[vec_inf/config](vec_inf/config/)`, then install from source by running `pip install .`. -- The package would try to look for cached configuration files in your environment before using the default configuration. The default cached configuration directory path points to `/model-weights/vec-inf-shared`, you would need to create an `environment.yaml` and a `models.yaml` following the format of these files in `[vec_inf/config](vec_inf/config/)`. -- [OPTIONAL] The package could also look for an enviroment variable `VEC_INF_CONFIG_DIR`. You can put your `environment.yaml` and `models.yaml` in a directory of your choice and set the enviroment variable `VEC_INF_CONFIG_DIR` to point to that location. +* Clone the repository and update the `environment.yaml` and the `models.yaml` file in [`vec_inf/config`](vec_inf/config/), then install from source by running `pip install .`. +* The package would try to look for cached configuration files in your environment before using the default configuration. The default cached configuration directory path points to `/model-weights/vec-inf-shared`, you would need to create an `environment.yaml` and a `models.yaml` following the format of these files in [`vec_inf/config`](vec_inf/config/). +* [OPTIONAL] The package could also look for an enviroment variable `VEC_INF_CONFIG_DIR`. You can put your `environment.yaml` and `models.yaml` in a directory of your choice and set the enviroment variable `VEC_INF_CONFIG_DIR` to point to that location. ## Usage @@ -42,26 +39,24 @@ We will use the Llama 3.1 model as example, to launch an OpenAI compatible infer ```bash vec-inf launch Meta-Llama-3.1-8B-Instruct ``` - You should see an output like the following: - +launch_image **NOTE**: You can set the required fields in the environment configuration (`environment.yaml`), it's a mapping between required arguments and their corresponding environment variables. On the Vector **Killarney** Cluster environment, the required fields are: + * `--account`, `-A`: The Slurm account, this argument can be set to default by setting environment variable `VEC_INF_ACCOUNT`. + * `--work-dir`, `-D`: A working directory other than your home directory, this argument can be set to default by seeting environment variable `VEC_INF_WORK_DIR`. -- `--account`, `-A`: The Slurm account, this argument can be set to default by setting environment variable `VEC_INF_ACCOUNT`. -- `--work-dir`, `-D`: A working directory other than your home directory, this argument can be set to default by seeting environment variable `VEC_INF_WORK_DIR`. - -Models that are already supported by `vec-inf` would be launched using the cached configuration (set in [slurm_vars.py](vec_inf/client/slurm_vars.py)) or [default configuration](vec_inf/config/models.yaml). You can override these values by providing additional parameters. Use `vec-inf launch --help` to see the full list of parameters that can be overriden. You can also launch your own custom model as long as the model architecture is supported by the underlying inference engine. For detailed instructions on how to customize your model launch, check out the `[launch` command section in User Guide](https://vectorinstitute.github.io/vector-inference/latest/user_guide/#launch-command). During the launch process, relevant log files and scripts will be written to a log directory (default to `.vec-inf-logs` in your home directory), and a cache directory (`.vec-inf-cache`) will be created in your working directory (defaults to your home directory if not specified or required) for torch compile cache. +Models that are already supported by `vec-inf` would be launched using the cached configuration (set in [slurm_vars.py](vec_inf/client/slurm_vars.py)) or [default configuration](vec_inf/config/models.yaml). You can override these values by providing additional parameters. Use `vec-inf launch --help` to see the full list of parameters that can be overriden. You can also launch your own custom model as long as the model architecture is supported by the underlying inference engine. For detailed instructions on how to customize your model launch, check out the [`launch` command section in User Guide](https://vectorinstitute.github.io/vector-inference/latest/user_guide/#launch-command). During the launch process, relevant log files and scripts will be written to a log directory (default to `.vec-inf-logs` in your home directory), and a cache directory (`.vec-inf-cache`) will be created in your working directory (defaults to your home directory if not specified or required) for torch compile cache. #### Other commands -- `batch-launch`: Launch multiple model inference servers at once, currently ONLY single node models supported, -- `status`: Check the status of all `vec-inf` jobs, or a specific job by providing its job ID. -- `metrics`: Streams performance metrics to the console. -- `shutdown`: Shutdown a model by providing its Slurm job ID. -- `list`: List all available model names, or view the default/cached configuration of a specific model. -- `cleanup`: Remove old log directories, use `--help` to see the supported filters. Use `--dry-run` to preview what would be deleted. +* `batch-launch`: Launch multiple model inference servers at once, currently ONLY single node models supported, +* `status`: Check the status of all `vec-inf` jobs, or a specific job by providing its job ID. +* `metrics`: Streams performance metrics to the console. +* `shutdown`: Shutdown a model by providing its Slurm job ID. +* `list`: List all available model names, or view the default/cached configuration of a specific model. +* `cleanup`: Remove old log directories, use `--help` to see the supported filters. Use `--dry-run` to preview what would be deleted. For more details on the usage of these commands, refer to the [User Guide](https://vectorinstitute.github.io/vector-inference/user_guide/) @@ -78,8 +73,13 @@ client = VecInfClient() response = client.launch_model("Meta-Llama-3.1-8B-Instruct") job_id = response.slurm_job_id status = client.get_status(job_id) -if status == ModelStatus.READY: +if status.status == ModelStatus.READY: print(f"Model is ready at {status.base_url}") +# Alternatively, use wait_until_ready which will either return a StatusResponse or throw a ServerError +try: + status = wait_until_ready(job_id) + except ServerError as e: + print(f"Model launch failed: {e}") client.shutdown_model(job_id) ``` @@ -91,7 +91,7 @@ With every model launch, a Slurm script will be generated dynamically based on t ## Send inference requests -Once the inference server is ready, you can start sending in inference requests. We provide example scripts for sending inference requests in `[examples](examples)` folder. Make sure to update the model server URL and the model weights location in the scripts. For example, you can run `python examples/inference/llm/chat_completions.py`, and you should expect to see an output like the following: +Once the inference server is ready, you can start sending in inference requests. We provide example scripts for sending inference requests in [`examples`](examples) folder. Make sure to update the model server URL and the model weights location in the scripts. For example, you can run `python examples/inference/llm/chat_completions.py`, and you should expect to see an output like the following: ```json { @@ -125,23 +125,17 @@ Once the inference server is ready, you can start sending in inference requests. } ``` - **NOTE**: Certain models don't adhere to OpenAI's chat template, e.g. Mistral family. For these models, you can either change your prompt to follow the model's default chat template or provide your own chat template via `--chat-template: TEMPLATE_PATH`. ## SSH tunnel from your local device - If you want to run inference from your local device, you can open a SSH tunnel to your cluster environment like the following: - ```bash ssh -L 8081:10.1.1.29:8081 username@v.vectorinstitute.ai -N ``` - The example provided above is for the Vector Killarney cluster, change the variables accordingly for your environment. The IP address for the compute nodes on Killarney follow `10.1.1.XX` pattern, where `XX` is the GPU number (`kn029` -> `29` in this example). ## Reference - If you found Vector Inference useful in your research or applications, please cite using the following BibTeX template: - ``` @software{vector_inference, title = {Vector Inference: Efficient LLM inference on Slurm clusters}, @@ -152,4 +146,3 @@ If you found Vector Inference useful in your research or applications, please ci url = {https://github.com/VectorInstitute/vector-inference} } ``` - From 8cf7eebe335cda06e124e21e2e2fc15034ecc0ae Mon Sep 17 00:00:00 2001 From: Ananya Raval Date: Wed, 13 May 2026 18:55:56 -0400 Subject: [PATCH 6/6] Add changes in README.md --- README.md | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 8988eac6..a8c35954 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ For more details on the usage of these commands, refer to the [User Guide](https Example: ```python -from vec_inf.client import VecInfClient +ffrom vec_inf.client import VecInfClient from vec_inf.client.models import ModelStatus client = VecInfClient() @@ -73,13 +73,8 @@ client = VecInfClient() response = client.launch_model("Meta-Llama-3.1-8B-Instruct") job_id = response.slurm_job_id status = client.get_status(job_id) -if status.status == ModelStatus.READY: +if status == ModelStatus.READY: print(f"Model is ready at {status.base_url}") -# Alternatively, use wait_until_ready which will either return a StatusResponse or throw a ServerError -try: - status = wait_until_ready(job_id) - except ServerError as e: - print(f"Model launch failed: {e}") client.shutdown_model(job_id) ```