From 35373bb8d3b383c36ec156370524f8f550b0a549 Mon Sep 17 00:00:00 2001 From: Ari Date: Thu, 23 Mar 2023 11:07:05 -0400 Subject: [PATCH 1/2] Converted hard-coded port to use environment variable `SERVER_PORT` Added .gitignore to ignore generated and uploaded images --- .gitignore | 1 + visual_chatgpt.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..28433306 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +image/ diff --git a/visual_chatgpt.py b/visual_chatgpt.py index e9614894..9d0e7710 100644 --- a/visual_chatgpt.py +++ b/visual_chatgpt.py @@ -70,7 +70,7 @@ Thought: Do I need to use a tool? {agent_scratchpad}""" os.makedirs('image', exist_ok=True) - +server_port = int(os.getenv('SERVER_PORT', "1015")) def seed_everything(seed): random.seed(seed) @@ -1066,4 +1066,4 @@ def run_image(self, image, state, txt): clear.click(bot.memory.clear) clear.click(lambda: [], None, chatbot) clear.click(lambda: [], None, state) - demo.launch(server_name="0.0.0.0", server_port=1015) + demo.launch(server_name="0.0.0.0", server_port=server_port) From 485a27e874b7ec1350d862cbe0dd2503e929120e Mon Sep 17 00:00:00 2001 From: Ari Date: Thu, 23 Mar 2023 11:10:37 -0400 Subject: [PATCH 2/2] Updated README with SERVER_PORT option --- README.md | 98 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 54 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 4fd445f7..82fdd69c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Visual ChatGPT +# Visual ChatGPT **Visual ChatGPT** connects ChatGPT and a series of Visual Foundation Models to enable **sending** and **receiving** images during chatting. @@ -13,34 +13,33 @@ See our paper: [Visual ChatGPT: Talking, Drawing and Editing with V ## Updates: + - We propose the **template** idea in Visual ChatGPT! - - A template is a **pre-defined execution flow** that assists ChatGPT in assembling complex tasks involving multiple foundation models. - - A template contains the **experiential solution** to complex tasks as determined by humans. - - A template can **invoke multiple foundation models** or even **establish a new ChatGPT session** - - To define a **template**, simply adding a class with attributes `template_model = True` + - A template is a **pre-defined execution flow** that assists ChatGPT in assembling complex tasks involving multiple foundation models. + - A template contains the **experiential solution** to complex tasks as determined by humans. + - A template can **invoke multiple foundation models** or even **establish a new ChatGPT session** + - To define a **template**, simply adding a class with attributes `template_model = True` - Thanks to **@ShengmingYin** and **@thebestannie** for providing a template example in `InfinityOutPainting` class (see the following gif) - - Firstly, run `python visual_chatgpt.py --load "ImageCaptioning_cuda:0,ImageEditing_cuda:1,VisualQuestionAnswering_cuda:2"` - - Secondly, say `extend the image to 2048x1024` to Visual ChatGPT! - - By simply creating an `InfinityOutPainting` template, Visual ChatGPT can seamlessly extend images to any size through collaboration with existing `ImageCaptioning`, `ImageEditing`, and `VisualQuestionAnswering` foundation models, **without the need for additional training**. + - Firstly, run `python visual_chatgpt.py --load "ImageCaptioning_cuda:0,ImageEditing_cuda:1,VisualQuestionAnswering_cuda:2"` + - Secondly, say `extend the image to 2048x1024` to Visual ChatGPT! + - By simply creating an `InfinityOutPainting` template, Visual ChatGPT can seamlessly extend images to any size through collaboration with existing `ImageCaptioning`, `ImageEditing`, and `VisualQuestionAnswering` foundation models, **without the need for additional training**. - **Visual ChatGPT needs the effort of the community! We crave your contribution to add new and interesting features!** - - + ## Insight & Goal: + On the one hand, **ChatGPT (or LLMs)** serves as a **general interface** that provides a broad and diverse understanding of a wide range of topics. On the other hand, **Foundation Models** serve as **domain experts** by providing deep knowledge in specific domains. By leveraging **both general and deep knowledge**, we aim at building an AI that is capable of handling various tasks. +## Demo -## Demo -## System Architecture +## System Architecture -

Logo

- ## Quick Start ``` @@ -66,7 +65,7 @@ export OPENAI_API_KEY={Your_Private_Openai_Key} set OPENAI_API_KEY={Your_Private_Openai_Key} # Start Visual ChatGPT ! -# You can specify the GPU/CPU assignment by "--load", the parameter indicates which +# You can specify the GPU/CPU assignment by "--load", the parameter indicates which # Visual Foundation Model to use and where it will be loaded to # The model and device are separated by underline '_', the different models are separated by comma ',' # The available Visual Foundation Models can be found in the following table @@ -76,10 +75,10 @@ set OPENAI_API_KEY={Your_Private_Openai_Key} # Advice for CPU Users python visual_chatgpt.py --load ImageCaptioning_cpu,Text2Image_cpu -# Advice for 1 Tesla T4 15GB (Google Colab) +# Advice for 1 Tesla T4 15GB (Google Colab) python visual_chatgpt.py --load "ImageCaptioning_cuda:0,Text2Image_cuda:0" - -# Advice for 4 Tesla V100 32GB + +# Advice for 4 Tesla V100 32GB python visual_chatgpt.py --load "ImageCaptioning_cuda:0,ImageEditing_cuda:0, Text2Image_cuda:1,Image2Canny_cpu,CannyText2Image_cuda:1, Image2Depth_cpu,DepthText2Image_cuda:1,VisualQuestionAnswering_cuda:2, @@ -87,48 +86,59 @@ python visual_chatgpt.py --load "ImageCaptioning_cuda:0,ImageEditing_cuda:0, Image2Seg_cpu,SegText2Image_cuda:2,Image2Pose_cpu,PoseText2Image_cuda:2, Image2Hed_cpu,HedText2Image_cuda:3,Image2Normal_cpu, NormalText2Image_cuda:3,Image2Line_cpu,LineText2Image_cuda:3" - + ``` +## Run options + +Set the port to be running using the `SERVER_PORT` environment variable + +| Option | Description | Default | +| ----------- | ------------------------- | ------- | +| SERVER_PORT | Port to start gradio demo | 1015 | + ## GPU memory usage + Here we list the GPU memory usage of each visual foundation model, you can specify which one you like: | Foundation Model | GPU Memory (MB) | -|------------------------|-----------------| -| ImageEditing | 3981 | -| InstructPix2Pix | 2827 | -| Text2Image | 3385 | -| ImageCaptioning | 1209 | -| Image2Canny | 0 | -| CannyText2Image | 3531 | -| Image2Line | 0 | -| LineText2Image | 3529 | -| Image2Hed | 0 | -| HedText2Image | 3529 | -| Image2Scribble | 0 | -| ScribbleText2Image | 3531 | -| Image2Pose | 0 | -| PoseText2Image | 3529 | -| Image2Seg | 919 | -| SegText2Image | 3529 | -| Image2Depth | 0 | -| DepthText2Image | 3531 | -| Image2Normal | 0 | -| NormalText2Image | 3529 | -| VisualQuestionAnswering| 1495 | +| ----------------------- | --------------- | +| ImageEditing | 3981 | +| InstructPix2Pix | 2827 | +| Text2Image | 3385 | +| ImageCaptioning | 1209 | +| Image2Canny | 0 | +| CannyText2Image | 3531 | +| Image2Line | 0 | +| LineText2Image | 3529 | +| Image2Hed | 0 | +| HedText2Image | 3529 | +| Image2Scribble | 0 | +| ScribbleText2Image | 3531 | +| Image2Pose | 0 | +| PoseText2Image | 3529 | +| Image2Seg | 919 | +| SegText2Image | 3529 | +| Image2Depth | 0 | +| DepthText2Image | 3531 | +| Image2Normal | 0 | +| NormalText2Image | 3529 | +| VisualQuestionAnswering | 1495 | ## Acknowledgement + We appreciate the open source of the following projects: [Hugging Face](https://github.com/huggingface)   [LangChain](https://github.com/hwchase17/langchain)   -[Stable Diffusion](https://github.com/CompVis/stable-diffusion)   -[ControlNet](https://github.com/lllyasviel/ControlNet)   -[InstructPix2Pix](https://github.com/timothybrooks/instruct-pix2pix)   +[Stable Diffusion](https://github.com/CompVis/stable-diffusion)   +[ControlNet](https://github.com/lllyasviel/ControlNet)   +[InstructPix2Pix](https://github.com/timothybrooks/instruct-pix2pix)   [CLIPSeg](https://github.com/timojl/clipseg)   [BLIP](https://github.com/salesforce/BLIP)   ## Contact Information + For help or issues using the Visual ChatGPT, please submit a GitHub issue. For other communications, please contact Chenfei WU (chewu@microsoft.com) or Nan DUAN (nanduan@microsoft.com).