feat: add vision support for Google (#141)

This commit is contained in:
Drew Hintz
2024-10-13 17:40:03 -05:00
committed by GitHub
parent 6ea6d1448d
commit 9679f07d09
2 changed files with 16 additions and 4 deletions
@@ -7,7 +7,7 @@ from exchange import Message, Tool
from exchange.content import Text, ToolResult, ToolUse from exchange.content import Text, ToolResult, ToolUse
from exchange.providers.base import Provider, Usage from exchange.providers.base import Provider, Usage
from tenacity import retry, wait_fixed, stop_after_attempt from tenacity import retry, wait_fixed, stop_after_attempt
from exchange.providers.utils import raise_for_status, retry_if_status from exchange.providers.utils import raise_for_status, retry_if_status, encode_image
GOOGLE_HOST = "https://generativelanguage.googleapis.com/v1beta" GOOGLE_HOST = "https://generativelanguage.googleapis.com/v1beta"
@@ -111,9 +111,20 @@ class GoogleProvider(Provider):
elif isinstance(content, ToolUse): elif isinstance(content, ToolUse):
converted["parts"].append({"functionCall": {"name": content.name, "args": content.parameters}}) converted["parts"].append({"functionCall": {"name": content.name, "args": content.parameters}})
elif isinstance(content, ToolResult): elif isinstance(content, ToolResult):
converted["parts"].append( if content.output.startswith('"image:'):
{"functionResponse": {"name": content.tool_use_id, "response": {"content": content.output}}} image_path = content.output.replace('"image:', "").replace('"', "")
) converted["parts"].append(
{
"inline_data": {
"mime_type": "image/png",
"data": f"{encode_image(image_path)}",
}
}
)
else:
converted["parts"].append(
{"functionResponse": {"name": content.tool_use_id, "response": {"content": content.output}}}
)
messages_spec.append(converted) messages_spec.append(converted)
if not messages_spec: if not messages_spec:
@@ -9,6 +9,7 @@ from exchange.providers import get_provider
cases = [ cases = [
(get_provider("openai"), os.getenv("OPENAI_MODEL", "gpt-4o-mini")), (get_provider("openai"), os.getenv("OPENAI_MODEL", "gpt-4o-mini")),
(get_provider("google"), os.getenv("GOOGLE_MODEL", "gemini-1.5-flash")),
] ]