Adjust is-stuck check for the same steps to 3 until it's stopped (#2437)

opendevin/controller/agent_controller.py

@@ -334,28 +334,35 @@ class AgentController:
             )
         ]
 
-        if len(filtered_history) < 4:
+        if len(filtered_history) < 3:
             return False
 
         # FIXME rewrite this to be more readable
 
-        # Check if the last four (Action, Observation) tuples are too repetitive
-        last_four_tuples = filtered_history[-4:]
+        # Scenario 1: the same (Action, Observation) loop
+        # 3 pairs of (action, observation) to stop the agent
+        last_three_tuples = filtered_history[-3:]
 
         if all(
             # (Action, Observation) tuples
-            # compare the last action to the last four actions
-            self._eq_no_pid(last_four_tuples[-1][0], _tuple[0])
-            for _tuple in last_four_tuples
+            # compare the last action to the last three actions
+            self._eq_no_pid(last_three_tuples[-1][0], _tuple[0])
+            for _tuple in last_three_tuples
         ) and all(
-            # compare the last observation to the last four observations
-            self._eq_no_pid(last_four_tuples[-1][1], _tuple[1])
-            for _tuple in last_four_tuples
+            # compare the last observation to the last three observations
+            self._eq_no_pid(last_three_tuples[-1][1], _tuple[1])
+            for _tuple in last_three_tuples
         ):
             logger.warning('Action, Observation loop detected')
             return True
 
-        # (action, error) tuples
+        if len(filtered_history) < 4:
+            return False
+
+        last_four_tuples = filtered_history[-4:]
+
+        # Scenario 2: (action, error) pattern, not necessary identical error
+        # 4 pairs of (action, error) to stop the agent
         if all(
             self._eq_no_pid(last_four_tuples[-1][0], _tuple[0])
             for _tuple in last_four_tuples
@@ -369,7 +376,8 @@ class AgentController:
 
         # check if the agent repeats the same (Action, Observation)
         # every other step in the last six tuples
-
+        # step1 = step3 = step5
+        # step2 = step4 = step6
         if len(filtered_history) >= 6:
             last_six_tuples = filtered_history[-6:]
             if (

tests/unit/test_is_stuck.py

@@ -66,6 +66,7 @@ class TestAgentController:
             mock_warning.assert_called_once_with('Action, Observation loop detected')
 
     def test_is_stuck_repeating_action_error_observation(self, controller):
+        # (action, error_observation), not necessarily the same error
         message_action = MessageAction(content='Done', wait_for_response=False)
         message_action._source = EventSource.USER
         controller.state.history = [
@@ -185,14 +186,21 @@ class TestAgentController:
         ]
         assert controller._is_stuck() is False
 
-    def test_is_stuck_four_identical_tuples(self, controller):
+    def test_is_stuck_three_identical_tuples(self, controller):
+        # the same (action, observation), three times
+
+        # prepare messages to interrupt things
         message_action = MessageAction(content='Done', wait_for_response=False)
         message_action._source = EventSource.USER
+        message_action2 = MessageAction(content='Or not done?', wait_for_response=False)
+        message_action2._source = EventSource.USER
         controller.state.history = [
             (
                 MessageAction(content='Hello', wait_for_response=False),
                 Observation(content='Response 1'),
             ),
+            # message from the user shouldn't interfere with the detection
+            (message_action, NullObservation(content='')),
             (
                 CmdRunAction(command='ls'),
                 CmdOutputObservation(
@@ -207,19 +215,13 @@ class TestAgentController:
                 ),
             ),
             # message from the user shouldn't interfere with the detection
-            (message_action, NullObservation(content='')),
+            (message_action2, NullObservation(content='')),
             (
                 CmdRunAction(command='ls'),
                 CmdOutputObservation(
                     command_id=3, command='ls', content='file1.txt\nfile2.txt'
                 ),
             ),
-            (
-                CmdRunAction(command='ls'),
-                CmdOutputObservation(
-                    command_id=4, command='ls', content='file1.txt\nfile2.txt'
-                ),
-            ),
         ]
         with patch('logging.Logger.warning') as mock_warning:
             assert controller._is_stuck() is True