Document, rename Agent* exceptions to LLM* (#2508)

* rename "Agent" exceptions to LLM*, document

* LLMResponseError

opendevin/controller/agent_controller.py

@@ -6,9 +6,9 @@ from opendevin.controller.agent import Agent
 from opendevin.controller.state.state import State
 from opendevin.core.config import config
 from opendevin.core.exceptions import (
-    AgentMalformedActionError,
-    AgentNoActionError,
-    LLMOutputError,
+    LLMMalformedActionError,
+    LLMNoActionError,
+    LLMResponseError,
     MaxCharsExceedError,
 )
 from opendevin.core.logger import opendevin_logger as logger
@@ -113,6 +113,13 @@ class AgentController:
                 await self.set_agent_state_to(AgentState.ERROR)
 
     async def report_error(self, message: str, exception: Exception | None = None):
+        """
+        This error will be reported to the user and sent to the LLM next step, in the hope it can self-correct.
+
+        This method should be called for a particular type of errors:
+        - the string message should be user-friendly, it will be shown in the UI
+        - an ErrorObservation can be sent to the LLM by the agent, with the exception message, so it can self-correct next time
+        """
         self.state.error = message
         if exception:
             self.state.error += f': {str(exception)}'
@@ -293,8 +300,10 @@ class AgentController:
         try:
             action = self.agent.step(self.state)
             if action is None:
-                raise AgentNoActionError('No action was returned')
-        except (AgentMalformedActionError, AgentNoActionError, LLMOutputError) as e:
+                raise LLMNoActionError('No action was returned')
+        except (LLMMalformedActionError, LLMNoActionError, LLMResponseError) as e:
+            # report to the user
+            # and send the underlying exception to the LLM for self-correction
             await self.report_error(str(e))
             return
 

opendevin/controller/state/task.py

@@ -1,5 +1,5 @@
 from opendevin.core.exceptions import (
-    AgentMalformedActionError,
+    LLMMalformedActionError,
     TaskInvalidStateError,
 )
 from opendevin.core.logger import opendevin_logger as logger
@@ -180,15 +180,15 @@ class RootTask(Task):
         if id == '':
             return self
         if len(self.subtasks) == 0:
-            raise AgentMalformedActionError('Task does not exist:' + id)
+            raise LLMMalformedActionError('Task does not exist:' + id)
         try:
             parts = [int(p) for p in id.split('.')]
         except ValueError:
-            raise AgentMalformedActionError('Invalid task id:' + id)
+            raise LLMMalformedActionError('Invalid task id:' + id)
         task: Task = self
         for part in parts:
             if part >= len(task.subtasks):
-                raise AgentMalformedActionError('Task does not exist:' + id)
+                raise LLMMalformedActionError('Task does not exist:' + id)
             task = task.subtasks[part]
         return task
 

opendevin/core/exceptions.py

@@ -35,11 +35,6 @@ class AgentNotRegisteredError(Exception):
         super().__init__(message)
 
 
-class LLMOutputError(Exception):
-    def __init__(self, message):
-        super().__init__(message)
-
-
 class SandboxInvalidBackgroundCommandError(Exception):
     def __init__(self, id=None):
         if id is not None:
@@ -71,12 +66,22 @@ class BrowserUnavailableException(Exception):
         super().__init__(message)
 
 
-# These exceptions get sent back to the LLM
-class AgentMalformedActionError(Exception):
+# This exception gets sent back to the LLM
+# It might be malformed JSON
+class LLMMalformedActionError(Exception):
     def __init__(self, message='Malformed response'):
         super().__init__(message)
 
 
-class AgentNoActionError(Exception):
+# This exception gets sent back to the LLM
+# For some reason, the agent did not return an action
+class LLMNoActionError(Exception):
     def __init__(self, message='Agent must return an action'):
         super().__init__(message)
+
+
+# This exception gets sent back to the LLM
+# The LLM output did not include an action, or the action was not the expected type
+class LLMResponseError(Exception):
+    def __init__(self, message='Failed to retrieve action from LLM response'):
+        super().__init__(message)

opendevin/core/utils/json.py

@@ -3,7 +3,7 @@ from datetime import datetime
 
 from json_repair import repair_json
 
-from opendevin.core.exceptions import LLMOutputError
+from opendevin.core.exceptions import LLMResponseError
 from opendevin.events.event import Event
 from opendevin.events.serialization import event_to_dict
 
@@ -50,7 +50,7 @@ def loads(json_str, **kwargs):
                     json_str = repair_json(response)
                     return json.loads(json_str, **kwargs)
                 except (json.JSONDecodeError, ValueError, TypeError) as e:
-                    raise LLMOutputError(
+                    raise LLMResponseError(
                         'Invalid JSON in response. Please make sure the response is a valid JSON object.'
                     ) from e
-    raise LLMOutputError('No valid JSON object found in response.')
+    raise LLMResponseError('No valid JSON object found in response.')

opendevin/events/serialization/action.py

@@ -1,4 +1,4 @@
-from opendevin.core.exceptions import AgentMalformedActionError
+from opendevin.core.exceptions import LLMMalformedActionError
 from opendevin.events.action.action import Action
 from opendevin.events.action.agent import (
     AgentDelegateAction,
@@ -42,22 +42,22 @@ ACTION_TYPE_TO_CLASS = {action_class.action: action_class for action_class in ac
 
 def action_from_dict(action: dict) -> Action:
     if not isinstance(action, dict):
-        raise AgentMalformedActionError('action must be a dictionary')
+        raise LLMMalformedActionError('action must be a dictionary')
     action = action.copy()
     if 'action' not in action:
-        raise AgentMalformedActionError(f"'action' key is not found in {action=}")
+        raise LLMMalformedActionError(f"'action' key is not found in {action=}")
     if not isinstance(action['action'], str):
-        raise AgentMalformedActionError(
+        raise LLMMalformedActionError(
             f"'{action['action']=}' is not defined. Available actions: {ACTION_TYPE_TO_CLASS.keys()}"
         )
     action_class = ACTION_TYPE_TO_CLASS.get(action['action'])
     if action_class is None:
-        raise AgentMalformedActionError(
+        raise LLMMalformedActionError(
             f"'{action['action']=}' is not defined. Available actions: {ACTION_TYPE_TO_CLASS.keys()}"
         )
     args = action.get('args', {})
     try:
         decoded_action = action_class(**args)
     except TypeError:
-        raise AgentMalformedActionError(f'action={action} has the wrong arguments')
+        raise LLMMalformedActionError(f'action={action} has the wrong arguments')
     return decoded_action

tests/unit/test_response_parsing.py

@@ -5,7 +5,7 @@ from agenthub.monologue_agent.utils.prompts import (
     parse_action_response as parse_response_monologue,
 )
 from agenthub.planner_agent.prompt import parse_response as parse_response_planner
-from opendevin.core.exceptions import LLMOutputError
+from opendevin.core.exceptions import LLMResponseError
 from opendevin.core.utils.json import loads as custom_loads
 from opendevin.events.action import (
     FileWriteAction,
@@ -86,11 +86,11 @@ def test_parse_first_of_multiple_jsons(parse_response_module):
 def test_invalid_json_raises_error():
     # This should fail if repair_json is able to fix this faulty JSON
     input_response = '{"action": "write", "args": { "path": "./short_essay.txt", "content": "Missing closing brace" }'
-    with pytest.raises(LLMOutputError):
+    with pytest.raises(LLMResponseError):
         custom_loads(input_response)
 
 
 def test_no_json_found():
     input_response = 'This is just a string with no JSON object.'
-    with pytest.raises(LLMOutputError):
+    with pytest.raises(LLMResponseError):
         custom_loads(input_response)