OpenHands/.github/workflows/run-integration-tests.yml

name: Run Integration Tests

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

on:
  push:
    branches:
    - main
  pull_request:

jobs:
  integration-tests:
    name: Integration Tests
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.11"]
        agent: ["SWEAgent", "PlannerAgent", "MonologueAgent", "CodeActAgent"]
        sandbox: ["ssh", "exec"]
        include:
          - agent: "MonologueAgent"
            embedding-model: "local"
          - agent: "MonologueAgent"
            # sufficient to have one agent testing against local sandbox
            sandbox: "local"
            embedding-model: "local"
          - agent: "SWEAgent"
            embedding-model: "none"
          - agent: "PlannerAgent"
            embedding-model: "none"
          - agent: "CodeActAgent"
            embedding-model: "none"
    steps:
      - uses: actions/checkout@v4

      - name: Install poetry via pipx
        run: pipx install poetry

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
          cache: 'poetry'

      - name: Install Python dependencies using Poetry
        run: poetry install

      - name: Build Environment
        run: make build

      - name: Run Integration Tests
        env:
          SANDBOX_TYPE: ${{ matrix.sandbox }}
          AGENT: ${{ matrix.agent }}
          MAX_ITERATIONS: 10
          LLM_EMBEDDING_MODEL: ${{ matrix.embedding-model }}
        run: |
          rm -rf workspace
          mkdir workspace
          WORKSPACE_BASE="$GITHUB_WORKSPACE/workspace" \
            WORKSPACE_MOUNT_PATH="$GITHUB_WORKSPACE/workspace" \
            poetry run pytest --cov=agenthub --cov=opendevin --cov-report=xml \
            -s ./tests/integration
      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v4
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
  test_matrix_success:
    name: All Integration Tests Passed
    runs-on: ubuntu-latest
    needs: [integration-tests]
    steps:
    - run: echo Done!