OpenHands/.github/workflows/run-integration-tests.yml

name: Run Integration Tests

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

on:
  push:
    branches:
      - main
    paths-ignore:
      - '**/*.md'
      - 'frontend/**'
      - 'docs/**'
      - 'evaluation/**'
  pull_request:

env:
  PERSIST_SANDBOX : "false"

jobs:
  integration-tests-on-linux:
    name: Integration Tests on Linux
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.11"]
        sandbox: ["ssh", "exec", "local"]
    steps:
      - uses: actions/checkout@v4

      - name: Install poetry via pipx
        run: pipx install poetry

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: 'poetry'

      - name: Install Python dependencies using Poetry
        run: poetry install

      - name: Build Environment
        run: make build

      - name: Run Integration Tests
        env:
          SANDBOX_TYPE: ${{ matrix.sandbox }}
        run: |
          TEST_IN_CI=true TEST_ONLY=true ./tests/integration/regenerate.sh

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v4
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

  integration-tests-on-mac:
    name: Integration Tests on MacOS
    runs-on: macos-13
    if: contains(github.event.pull_request.title, 'mac') || contains(github.event.pull_request.title, 'Mac')
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.11"]
        sandbox: ["ssh"]
    steps:
      - uses: actions/checkout@v4

      - name: Install poetry via pipx
        run: pipx install poetry

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: 'poetry'

      - name: Install Python dependencies using Poetry
        run: poetry install

      - name: Install & Start Docker
        run: |
          brew install colima docker
          colima start

          # For testcontainers to find the Colima socket
          # https://github.com/abiosoft/colima/blob/main/docs/FAQ.md#cannot-connect-to-the-docker-daemon-at-unixvarrundockersock-is-the-docker-daemon-running
          sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock

      - name: Build Environment
        run: make build

      - name: Run Integration Tests
        env:
          SANDBOX_TYPE: ${{ matrix.sandbox }}
        run: |
          TEST_IN_CI=true TEST_ONLY=true ./tests/integration/regenerate.sh

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v4
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}