Use of reasoning models for test metric generation. #2263
Description
I have checked the documentation and related resources and couldn't resolve my bug.
I am trying to use reasoning model(gpt-5-mini), for generating the metrics. But the langchain wrapper is seeming to take a default value of 0.5 or 0.01 which does not seem overridable using the temperature in langchain. Thus I am not able to use reasoning models. Is there any work around for this or can we expect a fix in the next release.
Ragas version: 0.3.3
Python version: 3.12
Code to Reproduce
from ragas import SingleTurnSample
from langchain_openai.chat_models import AzureChatOpenAI
from langchain_openai.embeddings import AzureOpenAIEmbeddings
from ragas.llms import LangchainLLMWrapper
from ragas.embeddings import LangchainEmbeddingsWrapper
from ragas.run_config import RunConfig
from llama_index.core import SimpleDirectoryReader
from langchain_community.document_loaders import DirectoryLoader
from ragas.testset import TestsetGenerator
from ragas.cost import get_token_usage_for_openai
from dotenv import load_dotenv
from azure.identity import ManagedIdentityCredential, get_bearer_token_provider
import os
from langchain_openai.chat_models import AzureChatOpenAI
import pandas as pd
load_dotenv()
## Set up Azure OpenAI with Managed Identity Credential in azureml workspace
os.environ["https_proxy"]=''
os.environ["http_proxy"]=''
credential = ManagedIdentityCredential(client_id=kv_client_id)
# Set OpenAI environment variables
os.environ["OPENAI_API_TYPE"] = "azure_ad"
os.environ["OPENAI_API_KEY"] = credential.get_token(
"https://cognitiveservices.azure.com/.default"
).token
# Initialize Bearer token provider
token_provider = get_bearer_token_provider(
credential, ""
)
azure_openai_configs = {
"base_url": "",
"api_version": "2025-01-01-preview",
"model_deployment": "gpt-5-mini-dzs",
"model_name": "gpt-5-mini",
"embedding_deployment": "text-embedding-ada-002",
"embedding_name": "text-embedding-ada-002",
}
azure_llm = AzureChatOpenAI(
openai_api_version=azure_openai_configs["api_version"],
azure_endpoint=azure_openai_configs["base_url"],
azure_deployment=azure_openai_configs["model_deployment"],
model=azure_openai_configs["model_name"],
azure_ad_token_provider=token_provider,
)
azure_embeddings = AzureOpenAIEmbeddings(
openai_api_version=api_version,
azure_endpoint=azure_openai_configs["base_url"],
azure_deployment=azure_openai_configs["embedding_deployment"],
model=azure_openai_configs["embedding_name"],
azure_ad_token_provider=token_provider,
)
azure_llm = LangchainLLMWrapper(azure_llm)
azure_embeddings = LangchainEmbeddingsWrapper(azure_embeddings)
my_run_config = RunConfig(max_workers=16, timeout=60)
sample = SingleTurnSample(user_input=user_input,
retrieved_contexts=retrieved_contexts,
response=response,
reference=reference,
)
scorer = Faithfulness(llm=azure_llm)
score = scorer.single_turn_score(sample)
Error trace
BadRequestError Traceback (most recent call last)
Cell In[12], [line 9](vscode-notebook-cell:?execution_count=12&line=9)
3 sample = SingleTurnSample(user_input=user_input,
4 retrieved_contexts=retrieved_contexts,
5 response=response,
6 reference=reference,
7 )
8 scorer = Faithfulness(llm=azure_llm)
----> [9](vscode-notebook-cell:?execution_count=12&line=9) score = scorer.single_turn_score(sample)
File /anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/ragas/metrics/base.py:486, in SingleTurnMetric.single_turn_score(self, sample, callbacks)
484 if not group_cm.ended:
485 rm.on_chain_error(e)
--> [486](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/ragas/metrics/base.py:486) raise e
487 else:
488 if not group_cm.ended:
File /anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/ragas/metrics/base.py:480, in SingleTurnMetric.single_turn_score(self, sample, callbacks)
476 raise ImportError(
477 "It seems like your running this in a jupyter-like environment. Please install nest_asyncio with `pip install nest_asyncio` to make it work."
478 )
479 loop = asyncio.get_event_loop()
--> [480](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/ragas/metrics/base.py:480) score = loop.run_until_complete(
481 self._single_turn_ascore(sample=sample, callbacks=group_cm)
482 )
483 except Exception as e:
484 if not group_cm.ended:
File /anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/nest_asyncio.py:98, in _patch_loop.<locals>.run_until_complete(self, future)
95 if not f.done():
96 raise RuntimeError(
97 'Event loop stopped before Future completed.')
---> [98](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/nest_asyncio.py:98) return f.result()
File /anaconda/envs/llminspector_env_dev/lib/python3.12/asyncio/futures.py:202, in Future.result(self)
200 self.__log_traceback = False
201 if self._exception is not None:
--> [202](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/asyncio/futures.py:202) raise self._exception.with_traceback(self._exception_tb)
203 return self._result
File /anaconda/envs/llminspector_env_dev/lib/python3.12/asyncio/tasks.py:314, in Task.__step_run_and_handle_result(***failed resolving arguments***)
310 try:
311 if exc is None:
312 # We use the `send` method directly, because coroutines
313 # don't have `__iter__` and `__next__` methods.
--> [314](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/asyncio/tasks.py:314) result = coro.send(None)
315 else:
316 result = coro.throw(exc)
File /anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/ragas/metrics/_faithfulness.py:200, in Faithfulness._single_turn_ascore(self, sample, callbacks)
196 async def _single_turn_ascore(
197 self, sample: SingleTurnSample, callbacks: Callbacks
198 ) -> float:
199 row = sample.to_dict()
--> [200](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/ragas/metrics/_faithfulness.py:200) return await self._ascore(row, callbacks)
File /anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/ragas/metrics/_faithfulness.py:208, in Faithfulness._ascore(self, row, callbacks)
203 """
204 returns the NLI score for each (q, c, a) pair
205 """
206 assert self.llm is not None, "LLM is not set"
--> [208](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/ragas/metrics/_faithfulness.py:208) statements = await self._create_statements(row, callbacks)
209 statements = statements.statements
210 if statements == []:
File /anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/ragas/metrics/_faithfulness.py:174, in Faithfulness._create_statements(self, row, callbacks)
171 text, question = row["response"], row["user_input"]
173 prompt_input = StatementGeneratorInput(question=question, answer=text)
--> [174](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/ragas/metrics/_faithfulness.py:174) statements = await self.statement_generator_prompt.generate(
175 llm=self.llm,
176 data=prompt_input,
177 callbacks=callbacks,
178 )
180 return statements
File /anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/ragas/prompt/pydantic_prompt.py:144, in PydanticPrompt.generate(self, llm, data, temperature, stop, callbacks, retries_left)
141 callbacks = callbacks or []
143 # this is just a special case of generate_multiple
--> [144](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/ragas/prompt/pydantic_prompt.py:144) output_single = await self.generate_multiple(
145 llm=llm,
146 data=data,
147 n=1,
148 temperature=temperature,
149 stop=stop,
150 callbacks=callbacks,
151 retries_left=retries_left,
152 )
153 return output_single[0]
File /anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/ragas/prompt/pydantic_prompt.py:220, in PydanticPrompt.generate_multiple(self, llm, data, n, temperature, stop, callbacks, retries_left)
217 else:
218 # This is a Ragas LLM - use generate()
219 ragas_llm = t.cast(BaseRagasLLM, llm)
--> [220](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/ragas/prompt/pydantic_prompt.py:220) resp = await ragas_llm.generate(
221 prompt_value,
222 n=n,
223 temperature=temperature,
224 stop=stop,
225 callbacks=prompt_cb,
226 )
228 output_models = []
229 parser = RagasOutputParser(pydantic_object=self.output_model)
File /anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/ragas/llms/base.py:116, in BaseRagasLLM.generate(self, prompt, n, temperature, stop, callbacks)
111 temperature = self.get_temperature(n)
113 agenerate_text_with_retry = add_async_retry(
114 self.agenerate_text, self.run_config
115 )
--> [116](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/ragas/llms/base.py:116) result = await agenerate_text_with_retry(
117 prompt=prompt,
118 n=n,
119 temperature=temperature,
120 stop=stop,
121 callbacks=callbacks,
122 )
124 # check there are no max_token issues
125 if not self.is_finished(result):
File /anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/tenacity/_asyncio.py:88, in AsyncRetrying.wraps.<locals>.async_wrapped(*args, **kwargs)
86 @functools.wraps(fn)
87 async def async_wrapped(*args: t.Any, **kwargs: t.Any) -> t.Any:
---> [88](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/tenacity/_asyncio.py:88) return await fn(*args, **kwargs)
File /anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/tenacity/_asyncio.py:47, in AsyncRetrying.__call__(self, fn, *args, **kwargs)
45 retry_state = RetryCallState(retry_object=self, fn=fn, args=args, kwargs=kwargs)
46 while True:
---> [47](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/tenacity/_asyncio.py:47) do = self.iter(retry_state=retry_state)
48 if isinstance(do, DoAttempt):
49 try:
File /anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/tenacity/__init__.py:325, in BaseRetrying.iter(self, retry_state)
323 retry_exc = self.retry_error_cls(fut)
324 if self.reraise:
--> [325](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/tenacity/__init__.py:325) raise retry_exc.reraise()
326 raise retry_exc from fut.exception()
328 if self.wait:
File /anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/tenacity/__init__.py:158, in RetryError.reraise(self)
156 def reraise(self) -> t.NoReturn:
157 if self.last_attempt.failed:
--> [158](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/tenacity/__init__.py:158) raise self.last_attempt.result()
159 raise self
File /anaconda/envs/llminspector_env_dev/lib/python3.12/concurrent/futures/_base.py:449, in Future.result(self, timeout)
447 raise CancelledError()
448 elif self._state == FINISHED:
--> [449](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/concurrent/futures/_base.py:449) return self.__get_result()
451 self._condition.wait(timeout)
453 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
File /anaconda/envs/llminspector_env_dev/lib/python3.12/concurrent/futures/_base.py:401, in Future.__get_result(self)
399 if self._exception:
400 try:
--> [401](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/concurrent/futures/_base.py:401) raise self._exception
402 finally:
403 # Break a reference cycle with the exception in self._exception
404 self = None
File /anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/tenacity/_asyncio.py:50, in AsyncRetrying.__call__(self, fn, *args, **kwargs)
48 if isinstance(do, DoAttempt):
49 try:
---> [50](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/tenacity/_asyncio.py:50) result = await fn(*args, **kwargs)
51 except BaseException: # noqa: B902
52 retry_state.set_exception(sys.exc_info()) # type: ignore[arg-type]
File /anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/ragas/llms/base.py:264, in LangchainLLMWrapper.agenerate_text(self, prompt, n, temperature, stop, callbacks)
262 if hasattr(self.langchain_llm, "n"):
263 self.langchain_llm.n = n # type: ignore
--> [264](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/ragas/llms/base.py:264) result = await self.langchain_llm.agenerate_prompt(
265 prompts=[prompt],
266 stop=stop,
267 callbacks=callbacks,
268 )
269 else:
270 result = await self.langchain_llm.agenerate_prompt(
271 prompts=[prompt] * n,
272 stop=stop,
273 callbacks=callbacks,
274 )
File /anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/langchain_core/language_models/chat_models.py:1030, in BaseChatModel.agenerate_prompt(self, prompts, stop, callbacks, **kwargs)
1021 @override
1022 async def agenerate_prompt(
1023 self,
(...) 1027 **kwargs: Any,
1028 ) -> LLMResult:
1029 prompt_messages = [p.to_messages() for p in prompts]
-> [1030](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/langchain_core/language_models/chat_models.py:1030) return await self.agenerate(
1031 prompt_messages, stop=stop, callbacks=callbacks, **kwargs
1032 )
File /anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/langchain_core/language_models/chat_models.py:988, in BaseChatModel.agenerate(self, messages, stop, callbacks, tags, metadata, run_name, run_id, **kwargs)
975 if run_managers:
976 await asyncio.gather(
977 *[
978 run_manager.on_llm_end(
(...) 986 ]
987 )
--> [988](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/langchain_core/language_models/chat_models.py:988) raise exceptions[0]
989 flattened_outputs = [
990 LLMResult(generations=[res.generations], llm_output=res.llm_output) # type: ignore[union-attr]
991 for res in results
992 ]
993 llm_output = self._combine_llm_outputs([res.llm_output for res in results]) # type: ignore[union-attr]
File /anaconda/envs/llminspector_env_dev/lib/python3.12/asyncio/tasks.py:314, in Task.__step_run_and_handle_result(***failed resolving arguments***)
310 try:
311 if exc is None:
312 # We use the `send` method directly, because coroutines
313 # don't have `__iter__` and `__next__` methods.
--> [314](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/asyncio/tasks.py:314) result = coro.send(None)
315 else:
316 result = coro.throw(exc)
File /anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/langchain_core/language_models/chat_models.py:1158, in BaseChatModel._agenerate_with_cache(self, messages, stop, run_manager, **kwargs)
1156 result = generate_from_stream(iter(chunks))
1157 elif inspect.signature(self._agenerate).parameters.get("run_manager"):
-> [1158](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/langchain_core/language_models/chat_models.py:1158) result = await self._agenerate(
1159 messages, stop=stop, run_manager=run_manager, **kwargs
1160 )
1161 else:
1162 result = await self._agenerate(messages, stop=stop, **kwargs)
File /anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/langchain_openai/chat_models/base.py:1429, in BaseChatOpenAI._agenerate(self, messages, stop, run_manager, **kwargs)
1427 if raw_response is not None and hasattr(raw_response, "http_response"):
1428 e.response = raw_response.http_response # type: ignore[attr-defined]
-> [1429](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/langchain_openai/chat_models/base.py:1429) raise e
1430 if (
1431 self.include_response_headers
1432 and raw_response is not None
1433 and hasattr(raw_response, "headers")
1434 ):
1435 generation_info = {"headers": dict(raw_response.headers)}
File /anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/langchain_openai/chat_models/base.py:1422, in BaseChatOpenAI._agenerate(self, messages, stop, run_manager, **kwargs)
1415 return _construct_lc_result_from_responses_api(
1416 response,
1417 schema=original_schema_obj,
1418 metadata=generation_info,
1419 output_version=self.output_version,
1420 )
1421 else:
-> [1422](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/langchain_openai/chat_models/base.py:1422) raw_response = await self.async_client.with_raw_response.create(
1423 **payload
1424 )
1425 response = raw_response.parse()
1426 except Exception as e:
File /anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/openai/_legacy_response.py:381, in async_to_raw_response_wrapper.<locals>.wrapped(*args, **kwargs)
377 extra_headers[RAW_RESPONSE_HEADER] = "true"
379 kwargs["extra_headers"] = extra_headers
--> [381](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/openai/_legacy_response.py:381) return cast(LegacyAPIResponse[R], await func(*args, **kwargs))
File /anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/openai/resources/chat/completions/completions.py:2583, in AsyncCompletions.create(self, messages, model, audio, frequency_penalty, function_call, functions, logit_bias, logprobs, max_completion_tokens, max_tokens, metadata, modalities, n, parallel_tool_calls, prediction, presence_penalty, prompt_cache_key, reasoning_effort, response_format, safety_identifier, seed, service_tier, stop, store, stream, stream_options, temperature, tool_choice, tools, top_logprobs, top_p, user, verbosity, web_search_options, extra_headers, extra_query, extra_body, timeout)
2537 @required_args(["messages", "model"], ["messages", "model", "stream"])
2538 async def create(
2539 self,
(...) 2580 timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
2581 ) -> ChatCompletion | AsyncStream[ChatCompletionChunk]:
2582 validate_response_format(response_format)
-> [2583](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/openai/resources/chat/completions/completions.py:2583) return await self._post(
2584 "/chat/completions",
2585 body=await async_maybe_transform(
2586 {
2587 "messages": messages,
2588 "model": model,
2589 "audio": audio,
2590 "frequency_penalty": frequency_penalty,
2591 "function_call": function_call,
2592 "functions": functions,
2593 "logit_bias": logit_bias,
2594 "logprobs": logprobs,
2595 "max_completion_tokens": max_completion_tokens,
2596 "max_tokens": max_tokens,
2597 "metadata": metadata,
2598 "modalities": modalities,
2599 "n": n,
2600 "parallel_tool_calls": parallel_tool_calls,
2601 "prediction": prediction,
2602 "presence_penalty": presence_penalty,
2603 "prompt_cache_key": prompt_cache_key,
2604 "reasoning_effort": reasoning_effort,
2605 "response_format": response_format,
2606 "safety_identifier": safety_identifier,
2607 "seed": seed,
2608 "service_tier": service_tier,
2609 "stop": stop,
2610 "store": store,
2611 "stream": stream,
2612 "stream_options": stream_options,
2613 "temperature": temperature,
2614 "tool_choice": tool_choice,
2615 "tools": tools,
2616 "top_logprobs": top_logprobs,
2617 "top_p": top_p,
2618 "user": user,
2619 "verbosity": verbosity,
2620 "web_search_options": web_search_options,
2621 },
2622 completion_create_params.CompletionCreateParamsStreaming
2623 if stream
2624 else completion_create_params.CompletionCreateParamsNonStreaming,
2625 ),
2626 options=make_request_options(
2627 extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
2628 ),
2629 cast_to=ChatCompletion,
2630 stream=stream or False,
2631 stream_cls=AsyncStream[ChatCompletionChunk],
2632 )
File /anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/openai/_base_client.py:[1794](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/openai/_base_client.py:1794), in AsyncAPIClient.post(self, path, cast_to, body, files, options, stream, stream_cls)
1780 async def post(
1781 self,
1782 path: str,
(...) 1789 stream_cls: type[_AsyncStreamT] | None = None,
1790 ) -> ResponseT | _AsyncStreamT:
1791 opts = FinalRequestOptions.construct(
1792 method="post", url=path, json_data=body, files=await async_to_httpx_files(files), **options
1793 )
-> 1794 return await self.request(cast_to, opts, stream=stream, stream_cls=stream_cls)
File /anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/openai/_base_client.py:[1594](https://vscode-remote+amlext-002b-002fsubscriptions-002f270ae3d6-002d5d7d-002d47b8-002db82d-002d17f1eea30f72-002fresourcegroups-002faicpdqabgbl0-002fproviders-002fmicrosoft-002emachinelearningservices-002fworkspaces-002faicpdevqabamlweu-002fcomputes-002fdevqabcpu2770-002dkp.vscode-resource.vscode-cdn.net/anaconda/envs/llminspector_env_dev/lib/python3.12/site-packages/openai/_base_client.py:1594), in AsyncAPIClient.request(self, cast_to, options, stream, stream_cls)
1591 await err.response.aread()
1593 log.debug("Re-raising status error")
-> 1594 raise self._make_status_error_from_response(err.response) from None
1596 break
1598 assert response is not None, "could not resolve response (should never happen)"
BadRequestError: Error code: 400 - {'error': {'message': "Unsupported value: 'temperature' does not support 0.01 with this model. Only the default (1) value is supported.", 'type': 'invalid_request_error', 'param': 'temperature', 'code': 'unsupported_value'}}
Expected behavior
I expect it to take the temperature set in the langchain in this case instead of setting it's own.