(test) Include message separators in mock prompts (#1855)

* Add message separator to prompts in tests

* DEMO: remove existing prompts for PlannerAgent

* Add results after prompt regeneration

opendevin/llm/llm.py

@@ -24,6 +24,8 @@ from opendevin.core.logger import opendevin_logger as logger
 
 __all__ = ['LLM']
 
+message_separator = '\n\n----------\n\n'
+
 
 class LLM:
     """
@@ -182,7 +184,7 @@ class LLM:
                 messages = args[1]
             debug_message = ''
             for message in messages:
-                debug_message += '\n\n----------\n\n' + message['content']
+                debug_message += message_separator + message['content']
             llm_prompt_logger.debug(debug_message)
             resp = completion_unwrapped(*args, **kwargs)
             message_back = resp['choices'][0]['message']['content']

tests/integration/conftest.py

@@ -7,6 +7,8 @@ from functools import partial
 import pytest
 from litellm import completion
 
+from opendevin.llm.llm import message_separator
+
 script_dir = os.path.dirname(os.path.realpath(__file__))
 workspace_path = os.getenv('WORKSPACE_BASE')
 
@@ -102,7 +104,7 @@ def mock_completion(*args, test_name, **kwargs):
     messages = kwargs['messages']
     message_str = ''
     for message in messages:
-        message_str += message['content']
+        message_str += message_separator + message['content']
     if os.environ.get('FORCE_APPLY_PROMPTS') == 'true':
         # this assumes all response_(*).log filenames are in numerical order, starting from one
         cur_id += 1

tests/integration/mock/PlannerAgent/test_edits/prompt_001.log

@@ -1,4 +1,8 @@
 
+
+----------
+
+
 # Task
 You're a diligent software engineer AI. You can't see, draw, or interact with a
 browser, but you can read and write files, and you can run commands, and you can think.

tests/integration/mock/PlannerAgent/test_edits/prompt_002.log

@@ -1,4 +1,8 @@
 
+
+----------
+
+
 # Task
 You're a diligent software engineer AI. You can't see, draw, or interact with a
 browser, but you can read and write files, and you can run commands, and you can think.

tests/integration/mock/PlannerAgent/test_edits/prompt_003.log

@@ -1,4 +1,8 @@
 
+
+----------
+
+
 # Task
 You're a diligent software engineer AI. You can't see, draw, or interact with a
 browser, but you can read and write files, and you can run commands, and you can think.

tests/integration/mock/PlannerAgent/test_edits/prompt_004.log

@@ -1,4 +1,8 @@
 
+
+----------
+
+
 # Task
 You're a diligent software engineer AI. You can't see, draw, or interact with a
 browser, but you can read and write files, and you can run commands, and you can think.

tests/integration/mock/PlannerAgent/test_edits/prompt_005.log

@@ -1,4 +1,8 @@
 
+
+----------
+
+
 # Task
 You're a diligent software engineer AI. You can't see, draw, or interact with a
 browser, but you can read and write files, and you can run commands, and you can think.

tests/integration/mock/PlannerAgent/test_edits/prompt_006.log

@@ -1,4 +1,8 @@
 
+
+----------
+
+
 # Task
 You're a diligent software engineer AI. You can't see, draw, or interact with a
 browser, but you can read and write files, and you can run commands, and you can think.

tests/integration/mock/PlannerAgent/test_edits/prompt_007.log

@@ -1,4 +1,8 @@
 
+
+----------
+
+
 # Task
 You're a diligent software engineer AI. You can't see, draw, or interact with a
 browser, but you can read and write files, and you can run commands, and you can think.

tests/integration/mock/PlannerAgent/test_edits/prompt_008.log

@@ -1,4 +1,8 @@
 
+
+----------
+
+
 # Task
 You're a diligent software engineer AI. You can't see, draw, or interact with a
 browser, but you can read and write files, and you can run commands, and you can think.

tests/integration/mock/PlannerAgent/test_edits/prompt_009.log

@@ -1,4 +1,8 @@
 
+
+----------
+
+
 # Task
 You're a diligent software engineer AI. You can't see, draw, or interact with a
 browser, but you can read and write files, and you can run commands, and you can think.

tests/integration/mock/PlannerAgent/test_edits/prompt_010.log

@@ -1,4 +1,8 @@
 
+
+----------
+
+
 # Task
 You're a diligent software engineer AI. You can't see, draw, or interact with a
 browser, but you can read and write files, and you can run commands, and you can think.

tests/integration/mock/PlannerAgent/test_write_simple_script/prompt_001.log

@@ -1,4 +1,8 @@
 
+
+----------
+
+
 # Task
 You're a diligent software engineer AI. You can't see, draw, or interact with a
 browser, but you can read and write files, and you can run commands, and you can think.

tests/integration/mock/PlannerAgent/test_write_simple_script/prompt_002.log

@@ -1,4 +1,8 @@
 
+
+----------
+
+
 # Task
 You're a diligent software engineer AI. You can't see, draw, or interact with a
 browser, but you can read and write files, and you can run commands, and you can think.

tests/integration/mock/PlannerAgent/test_write_simple_script/prompt_003.log

@@ -1,4 +1,8 @@
 
+
+----------
+
+
 # Task
 You're a diligent software engineer AI. You can't see, draw, or interact with a
 browser, but you can read and write files, and you can run commands, and you can think.

tests/integration/mock/PlannerAgent/test_write_simple_script/prompt_004.log

@@ -1,4 +1,8 @@
 
+
+----------
+
+
 # Task
 You're a diligent software engineer AI. You can't see, draw, or interact with a
 browser, but you can read and write files, and you can run commands, and you can think.

tests/integration/mock/PlannerAgent/test_write_simple_script/prompt_005.log

@@ -1,4 +1,8 @@
 
+
+----------
+
+
 # Task
 You're a diligent software engineer AI. You can't see, draw, or interact with a
 browser, but you can read and write files, and you can run commands, and you can think.