Allow the Agent to run uniittests for verification. (#3609)

* Allow the Agent to run uniittests for verification.

* minor bugfix - removed artifact

evaluation/aider_bench/helper.py

@@ -6,6 +6,7 @@ INSTRUCTIONS_ADDENDUM = """
 Use the above instructions to modify the supplied files: {signature_file}
 Don't change the names of existing functions or classes, as they may be referenced from other code like unit tests, etc.
 
+Use the test_file: {test_file}, to verify the correctness of your solution. DO NOT EDIT the test file.
 Only use standard python libraries, don't suggest installing any packages.
 """
 

evaluation/aider_bench/run_infer.py

@@ -85,6 +85,13 @@ async def initialize_runtime(
             file_path,
             '/workspace',
         )
+        file_path = os.path.join(tmpdir, f'{instance.instance_name}_test.py')
+        with open(file_path, 'w') as f:
+            f.write(instance.test)
+        await runtime.copy_to(
+            file_path,
+            '/workspace',
+        )
     logger.info(f"{'-' * 50} END Runtime Initialization Fn {'-' * 50}")
 
 
@@ -101,6 +108,7 @@ async def complete_runtime(
     logger.info(f"{'-' * 50} BEGIN Runtime Completion Fn {'-' * 50}")
     obs: CmdOutputObservation
 
+    # Rewriting the test file to ignore any changes Agent may have made.
     script_name = f'{instance.instance_name}_test.py'
     with tempfile.TemporaryDirectory() as tmpdir:
         file_path = os.path.join(tmpdir, script_name)
@@ -155,6 +163,7 @@ async def process_instance(
     instruction = instance.instruction
     instruction += INSTRUCTIONS_ADDENDUM.format(
         signature_file=f'{instance.instance_name}.py',
+        test_file=f'{instance.instance_name}_test.py',
     )
     instruction += (
         'IMPORTANT: You should ONLY interact with the environment provided '