Split container image build & push (#2456)

* Split container image build & push

* Code cleanup

* Cleanup

* Add back useless docker_build_success step to make CI happy

* Revert "Cleanup"

This reverts commit 2a260791a95a110a54335141d017a418397eecf9.

* Use fresh built sandbox image in integration test

* fix dependency

* DEBUG: only build

* Attempt to fix dependency

* Change dependency

* Combine both jobs

* Fix env

* Remove Mac integration tests as they are too unstable

* Move sandbox tests to ghcr

* Use loaded image

.github/workflows/ghcr.yml

@@ -1,4 +1,4 @@
-name: Publish Docker Image
+name: Build Publish and Test Docker Image
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -7,7 +7,7 @@ concurrency:
 on:
   push:
     branches:
-    - main
+      - main
     tags:
       - '*'
   pull_request:
@@ -19,19 +19,23 @@ on:
         default: ''
 
 jobs:
-  ghcr_build_and_push:
+  ghcr_build:
     runs-on: ubuntu-latest
 
+    outputs:
+      tags: ${{ steps.capture-tags.outputs.tags }}
+
     permissions:
       contents: read
       packages: write
 
     strategy:
       matrix:
-        image: ["app", "sandbox"]
+        image: ["sandbox", "opendevin"]
+        platform: ["amd64", "arm64"]
 
     steps:
-      - name: checkout
+      - name: Checkout
         uses: actions/checkout@v4
 
       - name: Free Disk Space (Ubuntu)
@@ -40,7 +44,6 @@ jobs:
           # this might remove tools that are actually needed,
           # if set to "true" but frees about 6 GB
           tool-cache: true
-
           # all of these default to true, but feel free to set to
           # "false" if necessary for your workflow
           android: true
@@ -57,26 +60,207 @@ jobs:
         id: buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Login to ghcr
-        uses: docker/login-action@v1
+      - name: Build and export image
+        id: build
+        run: ./containers/build.sh ${{ matrix.image }} ${{ github.repository_owner }} ${{ matrix.platform }}
+
+      - name: Capture tags
+        id: capture-tags
+        run: |
+          tags=$(cat tags.txt)
+          echo "tags=$tags"
+          echo "tags=$tags" >> $GITHUB_OUTPUT
+
+      - name: Upload Docker image as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.image }}-docker-image-${{ matrix.platform }}
+          path: /tmp/${{ matrix.image }}_image_${{ matrix.platform }}.tar
+
+  test-for-sandbox:
+    name: Test for Sandbox
+    runs-on: ubuntu-latest
+    needs: ghcr_build
+    env:
+      PERSIST_SANDBOX: "false"
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install poetry via pipx
+        run: pipx install poetry
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: "poetry"
+
+      - name: Install Python dependencies using Poetry
+        run: make install-python-dependencies
+
+      - name: Download sandbox Docker image
+        uses: actions/download-artifact@v4
+        with:
+          name: sandbox-docker-image-amd64
+          path: /tmp/
+
+      - name: Load sandbox image and run sandbox tests
+        run: |
+          # Load the Docker image and capture the output
+          output=$(docker load -i /tmp/sandbox_image_amd64.tar)
+
+          # Extract the image name from the output
+          image_name=$(echo "$output" | grep -oP 'Loaded image: \K.*')
+
+          # Print the full name of the image
+          echo "Loaded Docker image: $image_name"
+
+          SANDBOX_CONTAINER_IMAGE=$image_name poetry run pytest --cov=agenthub --cov=opendevin --cov-report=xml -s ./tests/unit/test_sandbox.py
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v4
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  integration-tests-on-linux:
+    name: Integration Tests on Linux
+    runs-on: ubuntu-latest
+    needs: ghcr_build
+    env:
+      PERSIST_SANDBOX: "false"
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11"]
+        sandbox: ["ssh", "exec", "local"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install poetry via pipx
+        run: pipx install poetry
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'poetry'
+
+      - name: Install Python dependencies using Poetry
+        run: make install-python-dependencies
+
+      - name: Download sandbox Docker image
+        uses: actions/download-artifact@v4
+        with:
+          name: sandbox-docker-image-amd64
+          path: /tmp/
+
+      - name: Load sandbox image and run integration tests
+        env:
+          SANDBOX_TYPE: ${{ matrix.sandbox }}
+        run: |
+          # Load the Docker image and capture the output
+          output=$(docker load -i /tmp/sandbox_image_amd64.tar)
+
+          # Extract the image name from the output
+          image_name=$(echo "$output" | grep -oP 'Loaded image: \K.*')
+
+          # Print the full name of the image
+          echo "Loaded Docker image: $image_name"
+
+          SANDBOX_CONTAINER_IMAGE=$image_name TEST_IN_CI=true TEST_ONLY=true ./tests/integration/regenerate.sh
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v4
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  ghcr_push:
+    runs-on: ubuntu-latest
+    # don't push if integration tests or sandbox tests fail
+    needs: [ghcr_build, integration-tests-on-linux, test-for-sandbox]
+    if: github.ref == 'refs/heads/main'
+
+    env:
+      tags: ${{ needs.ghcr_build.outputs.tags }}
+
+    permissions:
+      contents: read
+      packages: write
+
+    strategy:
+      matrix:
+        image: ["sandbox", "opendevin"]
+        platform: ["amd64", "arm64"]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Login to GHCR
+        uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build and push ${{ matrix.image }}
-        if: "!github.event.pull_request.head.repo.fork"
+      - name: Download Docker images
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ matrix.image }}-docker-image-${{ matrix.platform }}
+          path: /tmp/${{ matrix.platform }}
+
+      - name: Load images and push to registry
         run: |
-          ./containers/build.sh ${{ matrix.image }} ${{ github.repository_owner }} --push
+          mv /tmp/${{ matrix.platform }}/${{ matrix.image }}_image_${{ matrix.platform }}.tar .
+          loaded_image=$(docker load -i ${{ matrix.image }}_image_${{ matrix.platform }}.tar | grep "Loaded image:" | awk '{print $3}')
+          tags=$(echo ${tags} | tr ' ' '\n')
+          for tag in $tags; do
+            echo "tag = $tag"
+            docker tag $loaded_image ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}:${tag}_${{ matrix.platform }}
+            docker push ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}:${tag}_${{ matrix.platform }}
+          done
+
+  create_manifest:
+    runs-on: ubuntu-latest
+    needs: [ghcr_build, ghcr_push]
+    if: github.ref == 'refs/heads/main'
+
+    env:
+      tags: ${{ needs.ghcr_build.outputs.tags }}
+
+    strategy:
+      matrix:
+        image: ["sandbox", "opendevin"]
+
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Login to GHCR
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build ${{ matrix.image }}
-        if: "github.event.pull_request.head.repo.fork"
+      - name: Create and push multi-platform manifest
         run: |
-          ./containers/build.sh ${{ matrix.image }} ${{ github.repository_owner }}
+          tags=$(echo ${tags} | tr ' ' '\n')
+          for tag in $tags; do
+            echo 'tag = $tag'
+            docker buildx imagetools create --tag ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}:$tag \
+              ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}:${tag}_amd64 \
+              ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}:${tag}_arm64
+          done
 
+  # FIXME: an admin needs to mark this as non-mandatory, and then we can remove it
   docker_build_success:
     name: Docker Build Success
     runs-on: ubuntu-latest
-    needs: ghcr_build_and_push
+    needs: ghcr_build
     steps:
     - run: echo Done!

.github/workflows/run-integration-tests.yml

@@ -1,104 +0,0 @@
-name: Run Integration Tests
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
-
-on:
-  push:
-    branches:
-      - main
-    paths-ignore:
-      - '**/*.md'
-      - 'frontend/**'
-      - 'docs/**'
-      - 'evaluation/**'
-  pull_request:
-
-env:
-  PERSIST_SANDBOX : "false"
-
-jobs:
-  integration-tests-on-linux:
-    name: Integration Tests on Linux
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.11"]
-        sandbox: ["ssh", "exec", "local"]
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Install poetry via pipx
-        run: pipx install poetry
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'poetry'
-
-      - name: Install Python dependencies using Poetry
-        run: poetry install
-
-      - name: Build Environment
-        run: make build
-
-      - name: Run Integration Tests
-        env:
-          SANDBOX_TYPE: ${{ matrix.sandbox }}
-        run: |
-          TEST_IN_CI=true TEST_ONLY=true ./tests/integration/regenerate.sh
-
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v4
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-
-  integration-tests-on-mac:
-    name: Integration Tests on MacOS
-    runs-on: macos-13
-    if: contains(github.event.pull_request.title, 'mac') || contains(github.event.pull_request.title, 'Mac')
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.11"]
-        sandbox: ["ssh"]
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Install poetry via pipx
-        run: pipx install poetry
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'poetry'
-
-      - name: Install Python dependencies using Poetry
-        run: poetry install
-
-      - name: Install & Start Docker
-        run: |
-          brew install colima docker
-          colima start
-
-          # For testcontainers to find the Colima socket
-          # https://github.com/abiosoft/colima/blob/main/docs/FAQ.md#cannot-connect-to-the-docker-daemon-at-unixvarrundockersock-is-the-docker-daemon-running
-          sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
-
-      - name: Build Environment
-        run: make build
-
-      - name: Run Integration Tests
-        env:
-          SANDBOX_TYPE: ${{ matrix.sandbox }}
-        run: |
-          TEST_IN_CI=true TEST_ONLY=true ./tests/integration/regenerate.sh
-
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v4
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/run-unit-tests.yml

@@ -97,33 +97,3 @@ jobs:
         uses: codecov/codecov-action@v4
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-
-  test-for-sandbox:
-    name: Test for Sandbox
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Install poetry via pipx
-        run: pipx install poetry
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.11"
-          cache: "poetry"
-
-      - name: Install Python dependencies using Poetry
-        run: poetry install
-
-      - name: Build Environment
-        run: make build
-
-      - name: Run Integration Test for Sandbox
-        run: |
-          poetry run pytest --cov=agenthub --cov=opendevin --cov-report=xml -s ./tests/unit/test_sandbox.py
-
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v4
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

containers/build.sh

@@ -3,12 +3,9 @@ set -eo pipefail
 
 image_name=$1
 org_name=$2
-push=0
-if [[ $3 == "--push" ]]; then
-  push=1
-fi
+platform=$3
 
-echo -e "Building: $image_name"
+echo "Building: $image_name for platform: $platform"
 tags=()
 
 OPEN_DEVIN_BUILD_VERSION="dev"
@@ -19,49 +16,57 @@ cache_tag="$cache_tag_base"
 if [[ -n $GITHUB_REF_NAME ]]; then
   # check if ref name is a version number
   if [[ $GITHUB_REF_NAME =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-    major_version=$(echo $GITHUB_REF_NAME | cut -d. -f1)
-    minor_version=$(echo $GITHUB_REF_NAME | cut -d. -f1,2)
-    tags+=($major_version $minor_version)
+    major_version=$(echo "$GITHUB_REF_NAME" | cut -d. -f1)
+    minor_version=$(echo "$GITHUB_REF_NAME" | cut -d. -f1,2)
+    tags+=("$major_version" "$minor_version")
   fi
-  sanitized=$(echo $GITHUB_REF_NAME | sed 's/[^a-zA-Z0-9.-]\+/-/g')
+  sanitized=$(echo "$GITHUB_REF_NAME" | sed 's/[^a-zA-Z0-9.-]\+/-/g')
   OPEN_DEVIN_BUILD_VERSION=$sanitized
   cache_tag+="-${sanitized}"
-  tags+=($sanitized)
+  tags+=("$sanitized")
 fi
 echo "Tags: ${tags[@]}"
 
-dir=./containers/$image_name
-if [ ! -f $dir/Dockerfile ]; then
+if [[ "$image_name" == "opendevin" ]]; then
+  dir="./containers/app"
+else
+  dir="./containers/$image_name"
+fi
+
+if [[ ! -f "$dir/Dockerfile" ]]; then
   echo "No Dockerfile found"
   exit 1
 fi
-if [ ! -f $dir/config.sh ]; then
+if [[ ! -f "$dir/config.sh" ]]; then
   echo "No config.sh found for Dockerfile"
   exit 1
 fi
-source $dir/config.sh
+
+source "$dir/config.sh"
+
 if [[ -n "$org_name" ]]; then
   DOCKER_ORG="$org_name"
 fi
-DOCKER_REPOSITORY=$DOCKER_REGISTRY/$DOCKER_ORG/$DOCKER_IMAGE
+
+DOCKER_REPOSITORY="$DOCKER_REGISTRY/$DOCKER_ORG/$DOCKER_IMAGE"
 DOCKER_REPOSITORY=${DOCKER_REPOSITORY,,} # lowercase
 echo "Repo: $DOCKER_REPOSITORY"
 echo "Base dir: $DOCKER_BASE_DIR"
 
 args=""
-for tag in ${tags[@]}; do
+for tag in "${tags[@]}"; do
   args+=" -t $DOCKER_REPOSITORY:$tag"
 done
-if [[ $push -eq 1 ]]; then
-  args+=" --push"
-  args+=" --cache-to=type=registry,ref=$DOCKER_REPOSITORY:$cache_tag,mode=max"
-fi
+
+output_image="/tmp/${image_name}_image_${platform}.tar"
 
 docker buildx build \
   $args \
-  --build-arg OPEN_DEVIN_BUILD_VERSION=$OPEN_DEVIN_BUILD_VERSION \
-  --cache-from=type=registry,ref=$DOCKER_REPOSITORY:$cache_tag \
-  --cache-from=type=registry,ref=$DOCKER_REPOSITORY:$cache_tag_base-main \
-  --platform linux/amd64,linux/arm64 \
+  --build-arg OPEN_DEVIN_BUILD_VERSION="$OPEN_DEVIN_BUILD_VERSION" \
+  --platform linux/$platform \
   --provenance=false \
-  -f $dir/Dockerfile $DOCKER_BASE_DIR
+  -f "$dir/Dockerfile" \
+  --output type=docker,dest="$output_image" \
+  "$DOCKER_BASE_DIR"
+
+echo "${tags[*]}" > tags.txt