OpenHands/evaluation/SWE-bench/scripts/prepare_devin_outputs_for_evaluation.py

'''
Script used to convert devin's output into the desired json format for evaluation on SWE-bench

Usage:
    python prepare_devin_outputs_for_evaluation.py <setting>
    <setting> can be "passed", "failed", "all"

Outputs:
    two json files under evaluation/SWE-bench/data/

'''

#fetch devin's outputs into a json file for evaluation
import os
import sys
import json
import requests
from tqdm import tqdm

def get_devin_eval_output(setting):
    repo_url = 'CognitionAI/devin-swebench-results'
    folder_path = 'output_diffs'

    base_url = 'https://api.github.com/repos/'
    pass_api_url = f'{base_url}{repo_url}/contents/{folder_path}/pass'
    failed_api_url = f'{base_url}{repo_url}/contents/{folder_path}/fail'

    pass_files_info = []
    failed_files_info = []

    def get_files(api_url, subfolder_name, files_info):
        response = requests.get(api_url)
        if response.status_code == 200:
            contents = response.json()
            for item in tqdm(contents):
                if item['type'] == 'file':
                    file_url = f"https://raw.githubusercontent.com/{repo_url}/main/{folder_path}/{subfolder_name}/{item['name']}"
                    file_content = requests.get(file_url).text
                    instance_id = item['name'][:-9]
                    model_name = 'Devin'  # Update with actual model name
                    files_info.append({
                        'instance_id': instance_id,
                        'model_patch': file_content,
                        'model_name_or_path': model_name,
                        'pass_or_fail': subfolder_name
                    })

    if setting == 'passed' or setting == 'all':
        get_files(pass_api_url, 'pass', pass_files_info)
    if setting == 'failed' or setting == 'all':
        get_files(failed_api_url, 'fail', failed_files_info)

    script_dir = os.path.dirname(os.path.abspath(__file__))
    output_dir = os.path.join(script_dir, '../data/devin/')

    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    if setting == 'passed' or setting == 'all':
        with open(os.path.join(output_dir, 'devin_swe_passed.json'), 'w') as pass_file:
            json.dump(pass_files_info, pass_file, indent=4)

    if setting == 'failed' or setting == 'all':
        with open(os.path.join(output_dir, 'devin_swe_failed.json'), 'w') as fail_file:
            json.dump(failed_files_info, fail_file, indent=4)

    if setting == 'all':
        merged_output = pass_files_info + failed_files_info
        with open(os.path.join(output_dir, 'devin_swe_outputs.json'), 'w') as merge_file:
            json.dump(merged_output, merge_file, indent=4)


if __name__ == '__main__':
    if len(sys.argv) != 2:
        print('Usage: python script_name.py <setting>')
        sys.exit(1)

    setting = sys.argv[1]
    get_devin_eval_output(setting)