Merge pull request tjardoo#46 from tjardoo/chat-image-url-support

Add image url support to chat completion endpoint

Author: tjardoo

README.md

@@ -24,6 +24,7 @@ More information: [set API key](#set-api-key), [add proxy](#add-proxy), [rate li
   - [Delete fine-tune model](#delete-fine-tune-model)
 - Chat
   - [Create chat completion](#create-chat-completion)
+  - [Create chat completion with image](#create-chat-completion-with-image)
   - [Function calling](#function-calling)
 - Images
   - [Create image](#create-image)
@@ -135,6 +136,7 @@ Creates a model response for the given chat conversation.
 
 ```rust
 use openai_dive::v1::api::Client;
+use openai_dive::v1::models::Gpt4Engine;
 use openai_dive::v1::resources::chat::{ChatCompletionParameters, ChatMessage, Role};
 use std::env;
 
@@ -145,16 +147,16 @@ async fn main() {
     let client = Client::new(api_key);
 
     let parameters = ChatCompletionParameters {
-        model: "gpt-3.5-turbo-16k-0613".to_string(),
+        model: Gpt4Engine::Gpt41106Preview.to_string(),
         messages: vec![
             ChatMessage {
                 role: Role::User,
-                content: Some("Hello!".to_string()),
+                content: ChatMessageContent::Text("Hello!".to_string()),
                 ..Default::default()
             },
             ChatMessage {
                 role: Role::User,
-                content: Some("Where are you located?".to_string()),
+                content: ChatMessageContent::Text("What is the capital of Vietnam?".to_string()),
                 ..Default::default()
             },
         ],
@@ -170,6 +172,55 @@ async fn main() {
 
 More information: [Create chat completion](https://platform.openai.com/docs/api-reference/chat/create)
 
+### Create chat completion with image
+
+Creates a model response for the given chat conversation.
+
+```rust
+use openai_dive::v1::api::Client;
+use openai_dive::v1::models::Gpt4Engine;
+use openai_dive::v1::resources::chat::{ChatCompletionParameters, ChatMessage, Role};
+use std::env;
+
+#[tokio::main]
+async fn main() {
+    let api_key = env::var("OPENAI_API_KEY").expect("$OPENAI_API_KEY is not set");
+
+    let client = Client::new(api_key);
+
+    let parameters = ChatCompletionParameters {
+        model: Gpt4Engine::Gpt4VisionPreview.to_string(),
+        messages: vec![
+            ChatMessage {
+                role: Role::User,
+                content: ChatMessageContent::Text("What is in this image?".to_string()),
+                ..Default::default()
+            },
+            ChatMessage {
+                role: Role::User,
+                content: ChatMessageContent::ImageUrl(vec![ImageUrl {
+                    r#type: "image_url".to_string(),
+                    text: None,
+                    image_url: ImageUrlType {
+                        url: "https://images.unsplash.com/photo-1526682847805-721837c3f83b?w=640".to_string(),
+                        detail: None,
+                    },
+                }]),
+                ..Default::default()
+            },
+        ],
+        max_tokens: Some(50),
+        ..Default::default()
+    };
+
+    let result = client.chat().create(parameters).await.unwrap();
+
+    println!("{:#?}", result);
+}
+```
+
+More information: [Create chat completion](https://platform.openai.com/docs/api-reference/chat/create)
+
 ### Function calling
 
 In an API call, you can describe functions and have the model intelligently choose to output a JSON object containing arguments to call one or many functions. The Chat Completions API does not call the function; instead, the model generates JSON that you can use to call the function in your code.
@@ -179,12 +230,11 @@ In an API call, you can describe functions and have the model intelligently choo
 
 ```rust
 use openai_dive::v1::api::Client;
+use openai_dive::v1::models::Gpt4Engine;
 use openai_dive::v1::resources::chat::{
-    ChatCompletionFunction, ChatCompletionParameters, ChatCompletionTool, ChatCompletionToolChoice,
-    ChatCompletionToolChoiceFunction, ChatCompletionToolChoiceFunctionName, ChatCompletionToolType,
-    ChatMessage, Role,
+    ChatCompletionFunction, ChatCompletionParameters, ChatCompletionTool, ChatCompletionToolType, ChatMessage,
+    ChatMessageContent,
 };
-use openai_dive::v1::resources::shared::FinishReason;
 use rand::Rng;
 use serde::{Deserialize, Serialize};
 use serde_json::{json, Value};
@@ -195,22 +245,14 @@ async fn main() {
 
     let client = Client::new(api_key);
 
-    let mut messages = vec![ChatMessage {
-        content: Some("Give me a random number between 25 and 50?".to_string()),
+    let messages = vec![ChatMessage {
+        content: ChatMessageContent::Text("Give me a random number between 100 and no more than 150?".to_string()),
         ..Default::default()
     }];
 
     let parameters = ChatCompletionParameters {
-        model: "gpt-3.5-turbo-0613".to_string(),
+        model: Gpt4Engine::Gpt41106Preview.to_string(),
         messages: messages.clone(),
-        tool_choice: Some(ChatCompletionToolChoice::ChatCompletionToolChoiceFunction(
-            ChatCompletionToolChoiceFunction {
-                r#type: Some(ChatCompletionToolType::Function),
-                function: ChatCompletionToolChoiceFunctionName {
-                    name: "get_random_number".to_string(),
-                },
-            },
-        )),
         tools: Some(vec![ChatCompletionTool {
             r#type: ChatCompletionToolType::Function,
             function: ChatCompletionFunction {
@@ -221,7 +263,8 @@ async fn main() {
                     "properties": {
                         "min": {"type": "integer", "description": "Minimum value of the random number."},
                         "max": {"type": "integer", "description": "Maximum value of the random number."},
-                    }
+                    },
+                    "required": ["min", "max"],
                 }),
             },
         }]),
@@ -230,34 +273,22 @@ async fn main() {
 
     let result = client.chat().create(parameters).await.unwrap();
 
-    for choice in result.choices.iter() {
-        if choice.finish_reason == FinishReason::StopSequenceReached {
-            if let Some(tool_calls) = &choice.message.tool_calls {
-                for tool_call in tool_calls.iter() {
-                    let random_numbers =
-                        serde_json::from_str(&tool_call.function.arguments).unwrap();
-
-                    if tool_call.function.name == "get_random_number" {
-                        let random_number_result = get_random_number(random_numbers);
-
-                        messages.push(ChatMessage {
-                            role: Role::Function,
-                            content: Some(serde_json::to_string(&random_number_result).unwrap()),
-                            name: Some("get_random_number".to_string()),
-                            ..Default::default()
-                        });
-
-                        let parameters = ChatCompletionParameters {
-                            model: "gpt-3.5-turbo-0613".to_string(),
-                            messages: messages.clone(),
-                            ..Default::default()
-                        };
-
-                        let result = client.chat().create(parameters).await.unwrap();
-
-                        println!("{:#?}", result);
-                    }
-                }
+    let message = result.choices[0].message.clone();
+
+    if let Some(tool_calls) = message.tool_calls {
+        for tool_call in tool_calls {
+            let name = tool_call.function.name;
+            let arguments = tool_call.function.arguments;
+
+            if name == "get_random_number" {
+                let random_numbers: RandomNumber = serde_json::from_str(&arguments).unwrap();
+
+                println!("Min: {:?}", &random_numbers.min);
+                println!("Max: {:?}", &random_numbers.max);
+
+                let random_number_result = get_random_number(random_numbers);
+
+                println!("Random number between those numbers: {:?}", random_number_result.clone());
             }
         }
     }

examples/Cargo.toml

@@ -6,6 +6,7 @@ members = [
     "audio/create_speech",
     "chat/create_chat_completion",
     "chat/create_chat_completion_stream",
+    "chat/create_image_chat_completion",
     "chat/function_calling",
     "chat/function_calling_stream",
     "chat/rate_limit_headers",

examples/chat/Cargo.toml

@@ -3,6 +3,7 @@ resolver = "2"
 members = [
     "create_chat_completion",
     "create_chat_completion_stream",
+    "create_image_chat_completion",
     "function_calling",
     "function_calling_stream",
     "rate_limit_headers",

examples/chat/create_chat_completion/src/main.rs

@@ -1,5 +1,6 @@
 use openai_dive::v1::api::Client;
-use openai_dive::v1::resources::chat::{ChatCompletionParameters, ChatMessage, Role};
+use openai_dive::v1::models::Gpt4Engine;
+use openai_dive::v1::resources::chat::{ChatCompletionParameters, ChatMessage, ChatMessageContent, Role};
 use std::env;
 
 #[tokio::main]
@@ -9,16 +10,16 @@ async fn main() {
     let client = Client::new(api_key);
 
     let parameters = ChatCompletionParameters {
-        model: "gpt-3.5-turbo-16k-0613".to_string(),
+        model: Gpt4Engine::Gpt41106Preview.to_string(),
         messages: vec![
             ChatMessage {
                 role: Role::User,
-                content: Some("Hello!".to_string()),
+                content: ChatMessageContent::Text("Hello!".to_string()),
                 ..Default::default()
             },
             ChatMessage {
                 role: Role::User,
-                content: Some("Where are you located?".to_string()),
+                content: ChatMessageContent::Text("What is the capital of Vietnam?".to_string()),
                 ..Default::default()
             },
         ],

examples/chat/create_chat_completion_stream/src/main.rs

@@ -1,6 +1,7 @@
 use futures::StreamExt;
 use openai_dive::v1::api::Client;
-use openai_dive::v1::resources::chat::{ChatCompletionParameters, ChatMessage, Role};
+use openai_dive::v1::models::Gpt4Engine;
+use openai_dive::v1::resources::chat::{ChatCompletionParameters, ChatMessage, ChatMessageContent, Role};
 use std::env;
 
 #[tokio::main]
@@ -10,16 +11,16 @@ async fn main() {
     let client = Client::new(api_key);
 
     let parameters = ChatCompletionParameters {
-        model: "gpt-3.5-turbo-16k-0613".to_string(),
+        model: Gpt4Engine::Gpt41106Preview.to_string(),
         messages: vec![
             ChatMessage {
                 role: Role::User,
-                content: Some("Hello!".to_string()),
+                content: ChatMessageContent::Text("Hello!".to_string()),
                 ..Default::default()
             },
             ChatMessage {
                 role: Role::User,
-                content: Some("Where are you located?".to_string()),
+                content: ChatMessageContent::Text("What is the capital of Vietnam?".to_string()),
                 ..Default::default()
             },
         ],

examples/chat/create_image_chat_completion/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "create_image_chat_completion"
+version = "0.1.0"
+edition = "2021"
+publish = false
+
+[dependencies]
+openai_dive = { path = "./../../../../openai-client" }
+tokio = { version = "1.0", features = ["full"] }

examples/chat/create_image_chat_completion/src/main.rs

@@ -0,0 +1,40 @@
+use openai_dive::v1::api::Client;
+use openai_dive::v1::models::Gpt4Engine;
+use openai_dive::v1::resources::chat::{ChatCompletionParameters, ChatMessage, ChatMessageContent, ImageUrl, ImageUrlType, Role};
+use std::env;
+
+#[tokio::main]
+async fn main() {
+    let api_key = env::var("OPENAI_API_KEY").expect("$OPENAI_API_KEY is not set");
+
+    let client = Client::new(api_key);
+
+    let parameters = ChatCompletionParameters {
+        model: Gpt4Engine::Gpt4VisionPreview.to_string(),
+        messages: vec![
+            ChatMessage {
+                role: Role::User,
+                content: ChatMessageContent::Text("What is in this image?".to_string()),
+                ..Default::default()
+            },
+            ChatMessage {
+                role: Role::User,
+                content: ChatMessageContent::ImageUrl(vec![ImageUrl {
+                    r#type: "image_url".to_string(),
+                    text: None,
+                    image_url: ImageUrlType {
+                        url: "https://images.unsplash.com/photo-1526682847805-721837c3f83b?w=640".to_string(),
+                        detail: None,
+                    },
+                }]),
+                ..Default::default()
+            },
+        ],
+        max_tokens: Some(50),
+        ..Default::default()
+    };
+
+    let result = client.chat().create(parameters).await.unwrap();
+
+    println!("{:#?}", result);
+}