Tool calling with Gemini through Vertex AI fails because of argument with Union type #463
Description
Checked other resources
- I added a very descriptive title to this issue.
- I searched the LangGraph/LangChain documentation with the integrated search.
- I used the GitHub search to find a similar question and didn't find it.
- I am sure that this is a bug in LangGraph/LangChain rather than my code.
- I am sure this is better as an issue rather than a GitHub discussion, since this is a LangGraph bug and not a design question.
Example Code
from langchain_google_vertexai import ChatVertexAI
from typing import Annotated, Optional
from typing_extensions import TypedDict
from langgraph.graph import StateGraph, START, END
from langgraph.graph.message import add_messages
from langgraph.prebuilt import ToolNode, tools_condition
from langchain_core.tools import tool
from datetime import date, datetime
# The following environment variables have to be set
# os.environ["GOOGLE_API_KEY"] = loaded from .zshrc
# os.environ["LANGCHAIN_API_KEY"] = loaded from .zshrc
llm = ChatVertexAI(model="gemini-1.5-flash", max_retries=0)
@tool
def failing_tool(
mock_arg: Optional[date | datetime] = None,
) -> str:
"""Description doesn't really matter.
Returns:
A string
"""
return ""
tools = [failing_tool]
llm_with_tools = llm.bind_tools(tools)
class State(TypedDict):
messages: Annotated[list, add_messages]
graph_builder = StateGraph(State)
def chatbot(state: State):
return {"messages": [llm_with_tools.invoke(state["messages"])]}
graph_builder.add_node("chatbot", chatbot)
tool_node = ToolNode(tools=tools)
graph_builder.add_node("tools", tool_node)
graph_builder.add_conditional_edges(
"chatbot",
tools_condition,
)
graph_builder.add_edge("tools", "chatbot")
graph_builder.set_entry_point("chatbot")
graph = graph_builder.compile()
user_input = "Hi there!"
events = graph.stream({"messages": [("user", user_input)]}, {}, stream_mode="values")
for event in events:
event["messages"][-1].pretty_print()Error Message and Stack Trace (if applicable)
Key 'anyOf' is not supported in schema, ignoring
================================ Human Message =================================
Hi there!
---------------------------------------------------------------------------
_InactiveRpcError Traceback (most recent call last)
File ~/Development/.venv/lib/python3.12/site-packages/google/api_core/grpc_helpers.py:76, in _wrap_unary_errors.<locals>.error_remapped_callable(*args, **kwargs)
75 try:
---> 76 return callable_(*args, **kwargs)
77 except grpc.RpcError as exc:
File ~/Development/.venv/lib/python3.12/site-packages/grpc/_channel.py:1181, in _UnaryUnaryMultiCallable.__call__(self, request, timeout, metadata, credentials, wait_for_ready, compression)
1175 (
1176 state,
1177 call,
1178 ) = self._blocking(
1179 request, timeout, metadata, credentials, wait_for_ready, compression
1180 )
-> 1181 return _end_unary_response_blocking(state, call, False, None)
File ~/Development/.venv/lib/python3.12/site-packages/grpc/_channel.py:1006, in _end_unary_response_blocking(state, call, with_call, deadline)
1005 else:
-> 1006 raise _InactiveRpcError(state)
_InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
status = StatusCode.INVALID_ARGUMENT
details = "Unable to submit request because one or more function parameters didn't specify the schema type field. Learn more: https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/function-calling"
debug_error_string = "UNKNOWN:Error received from peer ipv4:172.217.18.202:443 {created_time:"2024-08-22T11:40:29.649254+02:00", grpc_status:3, grpc_message:"Unable to submit request because one or more function parameters didn\'t specify the schema type field. Learn more: https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/function-calling"}"
>
The above exception was the direct cause of the following exception:
InvalidArgument Traceback (most recent call last)
Cell In[1], line 60
57 user_input = "Hi there!"
59 events = graph.stream({"messages": [("user", user_input)]}, {}, stream_mode="values")
---> 60 for event in events:
61 event["messages"][-1].pretty_print()
File ~/Development/.venv/lib/python3.12/site-packages/langgraph/pregel/__init__.py:984, in Pregel.stream(self, input, config, stream_mode, output_keys, interrupt_before, interrupt_after, debug)
981 del fut, task
983 # panic on failure or timeout
--> 984 _panic_or_proceed(done, inflight, loop.step)
985 # don't keep futures around in memory longer than needed
986 del done, inflight, futures
File ~/Development/.venv/lib/python3.12/site-packages/langgraph/pregel/__init__.py:1385, in _panic_or_proceed(done, inflight, step, timeout_exc_cls)
1383 inflight.pop().cancel()
1384 # raise the exception
-> 1385 raise exc
1387 if inflight:
1388 # if we got here means we timed out
1389 while inflight:
1390 # cancel all pending tasks
File ~/Development/.venv/lib/python3.12/site-packages/langgraph/pregel/executor.py:60, in BackgroundExecutor.done(self, task)
58 def done(self, task: concurrent.futures.Future) -> None:
59 try:
---> 60 task.result()
61 except GraphInterrupt:
62 # This exception is an interruption signal, not an error
63 # so we don't want to re-raise it on exit
64 self.tasks.pop(task)
File /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/concurrent/futures/_base.py:449, in Future.result(self, timeout)
447 raise CancelledError()
448 elif self._state == FINISHED:
--> 449 return self.__get_result()
451 self._condition.wait(timeout)
453 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
File /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/concurrent/futures/_base.py:401, in Future.__get_result(self)
399 if self._exception:
400 try:
--> 401 raise self._exception
402 finally:
403 # Break a reference cycle with the exception in self._exception
404 self = None
File /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/concurrent/futures/thread.py:58, in _WorkItem.run(self)
55 return
57 try:
---> 58 result = self.fn(*self.args, **self.kwargs)
59 except BaseException as exc:
60 self.future.set_exception(exc)
File ~/Development/.venv/lib/python3.12/site-packages/langgraph/pregel/retry.py:25, in run_with_retry(task, retry_policy)
23 task.writes.clear()
24 # run the task
---> 25 task.proc.invoke(task.input, task.config)
26 # if successful, end
27 break
File ~/Development/.venv/lib/python3.12/site-packages/langchain_core/runnables/base.py:2873, in RunnableSequence.invoke(self, input, config, **kwargs)
2869 config = patch_config(
2870 config, callbacks=run_manager.get_child(f"seq:step:{i+1}")
2871 )
2872 if i == 0:
-> 2873 input = step.invoke(input, config, **kwargs)
2874 else:
2875 input = step.invoke(input, config)
File ~/Development/.venv/lib/python3.12/site-packages/langgraph/utils.py:102, in RunnableCallable.invoke(self, input, config, **kwargs)
100 if accepts_config(self.func):
101 kwargs["config"] = config
--> 102 ret = context.run(self.func, input, **kwargs)
103 if isinstance(ret, Runnable) and self.recurse:
104 return ret.invoke(input, config)
Cell In[1], line 41
40 def chatbot(state: State):
---> 41 return {"messages": [llm_with_tools.invoke(state["messages"])]}
File ~/Development/.venv/lib/python3.12/site-packages/langchain_core/runnables/base.py:5060, in RunnableBindingBase.invoke(self, input, config, **kwargs)
5054 def invoke(
5055 self,
5056 input: Input,
5057 config: Optional[RunnableConfig] = None,
5058 **kwargs: Optional[Any],
5059 ) -> Output:
-> 5060 return self.bound.invoke(
5061 input,
5062 self._merge_configs(config),
5063 **{**self.kwargs, **kwargs},
5064 )
File ~/Development/.venv/lib/python3.12/site-packages/langchain_core/language_models/chat_models.py:274, in BaseChatModel.invoke(self, input, config, stop, **kwargs)
263 def invoke(
264 self,
265 input: LanguageModelInput,
(...)
269 **kwargs: Any,
270 ) -> BaseMessage:
271 config = ensure_config(config)
272 return cast(
273 ChatGeneration,
--> 274 self.generate_prompt(
275 [self._convert_input(input)],
276 stop=stop,
277 callbacks=config.get("callbacks"),
278 tags=config.get("tags"),
279 metadata=config.get("metadata"),
280 run_name=config.get("run_name"),
281 run_id=config.pop("run_id", None),
282 **kwargs,
283 ).generations[0][0],
284 ).message
File ~/Development/.venv/lib/python3.12/site-packages/langchain_core/language_models/chat_models.py:714, in BaseChatModel.generate_prompt(self, prompts, stop, callbacks, **kwargs)
706 def generate_prompt(
707 self,
708 prompts: List[PromptValue],
(...)
711 **kwargs: Any,
712 ) -> LLMResult:
713 prompt_messages = [p.to_messages() for p in prompts]
--> 714 return self.generate(prompt_messages, stop=stop, callbacks=callbacks, **kwargs)
File ~/Development/.venv/lib/python3.12/site-packages/langchain_core/language_models/chat_models.py:571, in BaseChatModel.generate(self, messages, stop, callbacks, tags, metadata, run_name, run_id, **kwargs)
569 if run_managers:
570 run_managers[i].on_llm_error(e, response=LLMResult(generations=[]))
--> 571 raise e
572 flattened_outputs = [
573 LLMResult(generations=[res.generations], llm_output=res.llm_output) # type: ignore[list-item]
574 for res in results
575 ]
576 llm_output = self._combine_llm_outputs([res.llm_output for res in results])
File ~/Development/.venv/lib/python3.12/site-packages/langchain_core/language_models/chat_models.py:561, in BaseChatModel.generate(self, messages, stop, callbacks, tags, metadata, run_name, run_id, **kwargs)
558 for i, m in enumerate(messages):
559 try:
560 results.append(
--> 561 self._generate_with_cache(
562 m,
563 stop=stop,
564 run_manager=run_managers[i] if run_managers else None,
565 **kwargs,
566 )
567 )
568 except BaseException as e:
569 if run_managers:
File ~/Development/.venv/lib/python3.12/site-packages/langchain_core/language_models/chat_models.py:793, in BaseChatModel._generate_with_cache(self, messages, stop, run_manager, **kwargs)
791 else:
792 if inspect.signature(self._generate).parameters.get("run_manager"):
--> 793 result = self._generate(
794 messages, stop=stop, run_manager=run_manager, **kwargs
795 )
796 else:
797 result = self._generate(messages, stop=stop, **kwargs)
File ~/Development/.venv/lib/python3.12/site-packages/langchain_google_vertexai/chat_models.py:1137, in ChatVertexAI._generate(self, messages, stop, run_manager, stream, **kwargs)
1135 if not self._is_gemini_model:
1136 return self._generate_non_gemini(messages, stop=stop, **kwargs)
-> 1137 return self._generate_gemini(
1138 messages=messages,
1139 stop=stop,
1140 run_manager=run_manager,
1141 is_gemini=True,
1142 **kwargs,
1143 )
File ~/Development/.venv/lib/python3.12/site-packages/langchain_google_vertexai/chat_models.py:1294, in ChatVertexAI._generate_gemini(self, messages, stop, run_manager, **kwargs)
1286 def _generate_gemini(
1287 self,
1288 messages: List[BaseMessage],
(...)
1291 **kwargs: Any,
1292 ) -> ChatResult:
1293 request = self._prepare_request_gemini(messages=messages, stop=stop, **kwargs)
-> 1294 response = _completion_with_retry(
1295 self.prediction_client.generate_content,
1296 max_retries=self.max_retries,
1297 request=request,
1298 metadata=self.default_metadata,
1299 **kwargs,
1300 )
1301 return self._gemini_response_to_chat_result(response)
File ~/Development/.venv/lib/python3.12/site-packages/langchain_google_vertexai/chat_models.py:590, in _completion_with_retry(generation_method, max_retries, run_manager, **kwargs)
583 return generation_method(**kwargs)
585 params = (
586 {k: v for k, v in kwargs.items() if k in _allowed_params_prediction_service}
587 if kwargs.get("is_gemini")
588 else kwargs
589 )
--> 590 return _completion_with_retry_inner(
591 generation_method,
592 **params,
593 )
File ~/Development/.venv/lib/python3.12/site-packages/tenacity/__init__.py:336, in BaseRetrying.wraps.<locals>.wrapped_f(*args, **kw)
334 copy = self.copy()
335 wrapped_f.statistics = copy.statistics # type: ignore[attr-defined]
--> 336 return copy(f, *args, **kw)
File ~/Development/.venv/lib/python3.12/site-packages/tenacity/__init__.py:475, in Retrying.__call__(self, fn, *args, **kwargs)
473 retry_state = RetryCallState(retry_object=self, fn=fn, args=args, kwargs=kwargs)
474 while True:
--> 475 do = self.iter(retry_state=retry_state)
476 if isinstance(do, DoAttempt):
477 try:
File ~/Development/.venv/lib/python3.12/site-packages/tenacity/__init__.py:376, in BaseRetrying.iter(self, retry_state)
374 result = None
375 for action in self.iter_state.actions:
--> 376 result = action(retry_state)
377 return result
File ~/Development/.venv/lib/python3.12/site-packages/tenacity/__init__.py:418, in BaseRetrying._post_stop_check_actions.<locals>.exc_check(rs)
416 retry_exc = self.retry_error_cls(fut)
417 if self.reraise:
--> 418 raise retry_exc.reraise()
419 raise retry_exc from fut.exception()
File ~/Development/.venv/lib/python3.12/site-packages/tenacity/__init__.py:185, in RetryError.reraise(self)
183 def reraise(self) -> t.NoReturn:
184 if self.last_attempt.failed:
--> 185 raise self.last_attempt.result()
186 raise self
File /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/concurrent/futures/_base.py:449, in Future.result(self, timeout)
447 raise CancelledError()
448 elif self._state == FINISHED:
--> 449 return self.__get_result()
451 self._condition.wait(timeout)
453 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
File /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/concurrent/futures/_base.py:401, in Future.__get_result(self)
399 if self._exception:
400 try:
--> 401 raise self._exception
402 finally:
403 # Break a reference cycle with the exception in self._exception
404 self = None
File ~/Development/.venv/lib/python3.12/site-packages/tenacity/__init__.py:478, in Retrying.__call__(self, fn, *args, **kwargs)
476 if isinstance(do, DoAttempt):
477 try:
--> 478 result = fn(*args, **kwargs)
479 except BaseException: # noqa: B902
480 retry_state.set_exception(sys.exc_info()) # type: ignore[arg-type]
File ~/Development/.venv/lib/python3.12/site-packages/langchain_google_vertexai/chat_models.py:583, in _completion_with_retry.<locals>._completion_with_retry_inner(generation_method, **kwargs)
581 @retry_decorator
582 def _completion_with_retry_inner(generation_method: Callable, **kwargs: Any) -> Any:
--> 583 return generation_method(**kwargs)
File ~/Development/.venv/lib/python3.12/site-packages/google/cloud/aiplatform_v1beta1/services/prediction_service/client.py:2125, in PredictionServiceClient.generate_content(self, request, model, contents, retry, timeout, metadata)
2122 self._validate_universe_domain()
2124 # Send the request.
-> 2125 response = rpc(
2126 request,
2127 retry=retry,
2128 timeout=timeout,
2129 metadata=metadata,
2130 )
2132 # Done; return the response.
2133 return response
File ~/Development/.venv/lib/python3.12/site-packages/google/api_core/gapic_v1/method.py:131, in _GapicCallable.__call__(self, timeout, retry, compression, *args, **kwargs)
128 if self._compression is not None:
129 kwargs["compression"] = compression
--> 131 return wrapped_func(*args, **kwargs)
File ~/Development/.venv/lib/python3.12/site-packages/google/api_core/grpc_helpers.py:78, in _wrap_unary_errors.<locals>.error_remapped_callable(*args, **kwargs)
76 return callable_(*args, **kwargs)
77 except grpc.RpcError as exc:
---> 78 raise exceptions.from_grpc_error(exc) from exc
InvalidArgument: 400 Unable to submit request because one or more function parameters didn't specify the schema type field. Learn more: https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/function-callingDescription
Hello!
I'm going through the Customer Support Chatbot tutorial using Gemini through Vertex AI instead of Anthropic and it crashes during Part 1 - Example Conversation.
When the graph calls the LLM, it doesn't accept the request because of an error with the provided tool list.
From what I can tell, the search_flight tool definition isn't handled well because of its arguments of type Optional[date | datetime], specifically the union [date | datetime]. It seems their type doesn't get included in the tool description that is passed to the model so the model rejects it.
I've replicated it with the provided example code.
System Info
System Information
OS: Darwin
OS Version: Darwin Kernel Version 23.6.0: Mon Jul 29 21:13:00 PDT 2024; root:xnu-10063.141.2~1/RELEASE_X86_64
Python Version: 3.12.3 (v3.12.3:f6650f9ad7, Apr 9 2024, 08:18:48) [Clang 13.0.0 (clang-1300.0.29.30)]
Package Information
langchain_core: 0.2.28
langchain: 0.2.12
langchain_community: 0.2.11
langsmith: 0.1.96
langchain_chroma: 0.1.2
langchain_google_vertexai: 1.0.8
langchain_text_splitters: 0.2.2
langchainhub: 0.1.21
langgraph: 0.2.0
langserve: 0.2.2