From fbf8ae39f84d1958db97676ddb1268b92c9df356 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Mon, 22 Jan 2024 13:10:26 +0200
Subject: [PATCH] API: Allow content arrays for multimodal OpenAI requests
 (#5277)

---
 extensions/openai/completions.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/extensions/openai/completions.py b/extensions/openai/completions.py
index 35dd5d14..ce9cc102 100644
--- a/extensions/openai/completions.py
+++ b/extensions/openai/completions.py
@@ -144,6 +144,26 @@ def convert_history(history):
     user_input = ""
     system_message = ""
 
+    if any(isinstance(entry['content'], list) for entry in history):
+        new_history = []
+        for entry in history:
+            if isinstance(entry['content'], list):
+                image_url = None
+                content = None
+                for item in entry['content']:
+                    if not isinstance(item, dict):
+                        continue
+                    if item['type'] == 'image_url' and isinstance(item['image_url'], dict):
+                        image_url = item['image_url']['url']
+                    elif item['type'] == 'text' and isinstance(item['text'], str):
+                        content = item['text']
+                if image_url and content:
+                    new_history.append({"image_url": image_url, "role": "user"})
+                    new_history.append({"content": content, "role": "user"})
+            else:
+                new_history.append(entry)
+        history = new_history
+
     for entry in history:
         if "image_url" in entry:
             image_url = entry['image_url']