feat: ability to configure temperature and top-p sampling for llm generation (#1556)

Co-authored-by: Jim Su <jimsu@protonmail.com>

opendevin/core/config.py

@@ -42,6 +42,8 @@ DEFAULT_CONFIG: dict = {
     ConfigType.AGENT_MEMORY_ENABLED: False,
     ConfigType.LLM_TIMEOUT: None,
     ConfigType.LLM_MAX_RETURN_TOKENS: None,
+    ConfigType.LLM_TEMPERATURE: None,
+    ConfigType.LLM_TOP_P: None,
     # GPT-4 pricing is $10 per 1M input tokens. Since tokenization happens on LLM side,
     # we cannot easily count number of tokens, but we can count characters.
     # Assuming 5 characters per token, 5 million is a reasonable default limit.

opendevin/core/schema/config.py

@@ -2,6 +2,8 @@ from enum import Enum
 
 
 class ConfigType(str, Enum):
+    LLM_TOP_P = 'LLM_TOP_P'
+    LLM_TEMPERATURE = 'LLM_TEMPERATURE'
     LLM_MAX_RETURN_TOKENS = 'LLM_MAX_RETURN_TOKENS'
     LLM_TIMEOUT = 'LLM_TIMEOUT'
     LLM_API_KEY = 'LLM_API_KEY'

opendevin/llm/llm.py

@@ -27,6 +27,8 @@ LLM_RETRY_MIN_WAIT = config.get(ConfigType.LLM_RETRY_MIN_WAIT)
 LLM_RETRY_MAX_WAIT = config.get(ConfigType.LLM_RETRY_MAX_WAIT)
 LLM_TIMEOUT = config.get(ConfigType.LLM_TIMEOUT)
 LLM_MAX_RETURN_TOKENS = config.get(ConfigType.LLM_MAX_RETURN_TOKENS)
+LLM_TEMPERATURE = config.get(ConfigType.LLM_TEMPERATURE)
+LLM_TOP_P = config.get(ConfigType.LLM_TOP_P)
 
 
 class LLM:
@@ -45,6 +47,8 @@ class LLM:
         retry_max_wait=LLM_RETRY_MAX_WAIT,
         llm_timeout=LLM_TIMEOUT,
         llm_max_return_tokens=LLM_MAX_RETURN_TOKENS,
+        llm_temperature=LLM_TEMPERATURE,
+        llm_top_p=LLM_TOP_P,
     ):
         """
         Args:
@@ -80,6 +84,8 @@ class LLM:
             api_version=self.api_version,
             max_tokens=self.llm_max_return_tokens,
             timeout=self.llm_timeout,
+            temperature=llm_temperature,
+            top_p=llm_top_p,
         )
 
         completion_unwrapped = self._completion