feat(runtime): add versioned runtime image (base_name+oh_version) (#4574)

containers/build.sh

@@ -98,9 +98,9 @@ if [[ -n "$org_name" ]]; then
   DOCKER_ORG="$org_name"
 fi
 
-# If $DOCKER_IMAGE_HASH_TAG is set, add it to the tags
-if [[ -n "$DOCKER_IMAGE_HASH_TAG" ]]; then
-  tags+=("$DOCKER_IMAGE_HASH_TAG")
+# If $DOCKER_IMAGE_SOURCE_TAG is set, add it to the tags
+if [[ -n "$DOCKER_IMAGE_SOURCE_TAG" ]]; then
+  tags+=("$DOCKER_IMAGE_SOURCE_TAG")
 fi
 # If $DOCKER_IMAGE_TAG is set, add it to the tags
 if [[ -n "$DOCKER_IMAGE_TAG" ]]; then

containers/runtime/config.sh

@@ -4,4 +4,4 @@ DOCKER_BASE_DIR="./containers/runtime"
 DOCKER_IMAGE=runtime
 # These variables will be appended by the runtime_build.py script
 # DOCKER_IMAGE_TAG=
-# DOCKER_IMAGE_HASH_TAG=
+# DOCKER_IMAGE_SOURCE_TAG=

docs/modules/usage/architecture/runtime.md

@@ -70,14 +70,22 @@ Check out the [relevant code](https://github.com/All-Hands-AI/OpenHands/blob/mai
 
 ### Image Tagging System
 
-OpenHands uses a dual-tagging system for its runtime images to balance reproducibility with flexibility.
+OpenHands uses a three-tag system for its runtime images to balance reproducibility with flexibility.
 Tags may be in one of 2 formats:
 
-- **Generic**: `oh_v{openhands_version}_{16_digit_lock_hash}` (e.g.: `oh_v0.9.9_1234567890abcdef`)
-- **Specific**: `oh_v{openhands_version}_{16_digit_lock_hash}_{16_digit_source_hash}`
+- **Versioned Tag**: `oh_v{openhands_version}_{base_image}` (e.g.: `oh_v0.9.9_nikolaik_s_python-nodejs_t_python3.12-nodejs22`)
+- **Lock Tag**: `oh_v{openhands_version}_{16_digit_lock_hash}` (e.g.: `oh_v0.9.9_1234567890abcdef`)
+- **Source Tag**: `oh_v{openhands_version}_{16_digit_lock_hash}_{16_digit_source_hash}`
   (e.g.: `oh_v0.9.9_1234567890abcdef_1234567890abcdef`)
 
-#### Lock Hash
+
+#### Source Tag - Most Specific
+
+This is the first 16 digits of the MD5 of the directory hash for the source directory. This gives a hash
+for only the openhands source
+
+
+#### Lock Tag
 
 This hash is built from the first 16 digits of the MD5 of:
 - The name of the base image upon which the image was built (e.g.: `nikolaik/python-nodejs:python3.12-nodejs22`)
@@ -86,30 +94,30 @@ This hash is built from the first 16 digits of the MD5 of:
 
 This effectively gives a hash for the dependencies of Openhands independent of the source code.
 
-#### Source Hash
+#### Versioned Tag - Most Generic
 
-This is the first 16 digits of the MD5 of the directory hash for the source directory. This gives a hash
-for only the openhands source
+This tag is a concatenation of openhands version and the base image name (transformed to fit in tag standard).
 
 #### Build Process
 
 When generating an image...
 
-- OpenHands first checks whether an image with the same **Specific** tag exists. If there is such an image,
+- **No re-build**: OpenHands first checks whether an image with the same **most specific source tag** exists. If there is such an image,
   no build is performed - the existing image is used.
-- OpenHands next checks whether an image with the **Generic** tag exists. If there is such an image,
+- **Fastest re-build**: OpenHands next checks whether an image with the **generic lock tag** exists. If there is such an image,
   OpenHands builds a new image based upon it, bypassing all installation steps (like `poetry install` and
   `apt-get`) except a final operation to copy the current source code. The new image is tagged with a
-  **Specific** tag only.
-- If neither a **Specific** nor **Generic** tag exists, a brand new image is built based upon the base
-  image (Which is a slower operation). This new image is tagged with both the **Generic** and **Specific**
-  tags.
+  **source** tag only.
+- **Ok-ish re-build**: If neither a **source** nor **lock** tag exists, an image will be built based upon the **versioned** tag image.
+  In versioned tag image, most dependencies should already been installed hence saving time.
+- **Slowest re-build**: If all of the three tags don't exists, a brand new image is built based upon the base
+  image (Which is a slower operation). This new image is tagged with all the **source**, **lock**, and **versioned** tags.
 
-This dual-tagging approach allows OpenHands to efficiently manage both development and production environments.
+This tagging approach allows OpenHands to efficiently manage both development and production environments.
 
 1. Identical source code and Dockerfile always produce the same image (via hash-based tags)
 2. The system can quickly rebuild images when minor changes occur (by leveraging recent compatible images)
-3. The generic tag (e.g., `runtime:oh_v0.9.3_1234567890abcdef`) always points to the latest build for a particular base image and OpenHands version combination
+3. The **lock** tag (e.g., `runtime:oh_v0.9.3_1234567890abcdef`) always points to the latest build for a particular base image, dependency, and OpenHands version combination
 
 ## Runtime Plugin System
 

openhands/runtime/builder/docker.py

@@ -58,7 +58,7 @@ class DockerRuntimeBuilder(RuntimeBuilder):
             raise RuntimeError('Docker server version must be >= 18.09 to use BuildKit')
 
         target_image_hash_name = tags[0]
-        target_image_repo, target_image_hash_tag = target_image_hash_name.split(':')
+        target_image_repo, target_image_source_tag = target_image_hash_name.split(':')
         target_image_tag = tags[1].split(':')[1] if len(tags) > 1 else None
 
         buildx_cmd = [
@@ -160,9 +160,9 @@ class DockerRuntimeBuilder(RuntimeBuilder):
             )
 
         tags_str = (
-            f'{target_image_hash_tag}, {target_image_tag}'
+            f'{target_image_source_tag}, {target_image_tag}'
             if target_image_tag
-            else target_image_hash_tag
+            else target_image_source_tag
         )
         logger.info(
             f'Image {target_image_repo} with tags [{tags_str}] built successfully'

openhands/runtime/impl/modal/modal_runtime.py

@@ -18,6 +18,7 @@ from openhands.runtime.impl.eventstream.eventstream_runtime import (
 from openhands.runtime.plugins import PluginRequirement
 from openhands.runtime.utils.command import get_remote_startup_command
 from openhands.runtime.utils.runtime_build import (
+    BuildFromImageType,
     prep_build_folder,
 )
 from openhands.utils.async_utils import call_sync_from_async
@@ -183,7 +184,7 @@ class ModalRuntime(EventStreamRuntime):
             prep_build_folder(
                 build_folder=Path(build_folder),
                 base_image=base_container_image_id,
-                build_from_scratch=True,
+                build_from=BuildFromImageType.SCRATCH,
                 extra_deps=runtime_extra_deps,
             )
 

openhands/runtime/utils/runtime_build.py

@@ -4,6 +4,7 @@ import os
 import shutil
 import string
 import tempfile
+from enum import Enum
 from pathlib import Path
 from typing import List
 
@@ -17,20 +18,26 @@ from openhands.core.logger import openhands_logger as logger
 from openhands.runtime.builder import DockerRuntimeBuilder, RuntimeBuilder
 
 
+class BuildFromImageType(Enum):
+    SCRATCH = 'scratch'  # Slowest: Build from base image (no dependencies are reused)
+    VERSIONED = 'versioned'  # Medium speed: Reuse the most recent image with the same base image & OH version (a lot of dependencies are already installed)
+    LOCK = 'lock'  # Fastest: Reuse the most recent image with the exact SAME dependencies (lock files)
+
+
 def get_runtime_image_repo():
     return os.getenv('OH_RUNTIME_RUNTIME_IMAGE_REPO', 'ghcr.io/all-hands-ai/runtime')
 
 
 def _generate_dockerfile(
     base_image: str,
-    build_from_scratch: bool = True,
+    build_from: BuildFromImageType = BuildFromImageType.SCRATCH,
     extra_deps: str | None = None,
 ) -> str:
     """Generate the Dockerfile content for the runtime image based on the base image.
 
     Parameters:
     - base_image (str): The base image provided for the runtime image
-    - build_from_scratch (boolean): False implies most steps can be skipped (Base image is another openhands instance)
+    - build_from (BuildFromImageType): The build method for the runtime image.
     - extra_deps (str):
 
     Returns:
@@ -45,7 +52,8 @@ def _generate_dockerfile(
 
     dockerfile_content = template.render(
         base_image=base_image,
-        build_from_scratch=build_from_scratch,
+        build_from_scratch=build_from == BuildFromImageType.SCRATCH,
+        build_from_versioned=build_from == BuildFromImageType.VERSIONED,
         extra_deps=extra_deps if extra_deps is not None else '',
     )
     return dockerfile_content
@@ -157,25 +165,36 @@ def build_runtime_image_in_folder(
 ) -> str:
     runtime_image_repo, _ = get_runtime_image_repo_and_tag(base_image)
     lock_tag = f'oh_v{oh_version}_{get_hash_for_lock_files(base_image)}'
-    hash_tag = f'{lock_tag}_{get_hash_for_source_files()}'
-    hash_image_name = f'{runtime_image_repo}:{hash_tag}'
+    versioned_tag = (
+        # truncate the base image to 96 characters to fit in the tag max length (128 characters)
+        f'oh_v{oh_version}_{get_tag_for_versioned_image(base_image)}'
+    )
+    versioned_image_name = f'{runtime_image_repo}:{versioned_tag}'
+    source_tag = f'{lock_tag}_{get_hash_for_source_files()}'
+    hash_image_name = f'{runtime_image_repo}:{source_tag}'
 
     if force_rebuild:
-        logger.info(f'Force rebuild: [{runtime_image_repo}:{hash_tag}] from scratch.')
-        prep_build_folder(build_folder, base_image, True, extra_deps)
+        logger.info(f'Force rebuild: [{runtime_image_repo}:{source_tag}] from scratch.')
+        prep_build_folder(
+            build_folder,
+            base_image,
+            build_from=BuildFromImageType.SCRATCH,
+            extra_deps=extra_deps,
+        )
         if not dry_run:
             _build_sandbox_image(
                 build_folder,
                 runtime_builder,
                 runtime_image_repo,
-                hash_tag,
+                source_tag,
                 lock_tag,
+                versioned_tag,
                 platform,
             )
         return hash_image_name
 
     lock_image_name = f'{runtime_image_repo}:{lock_tag}'
-    build_from_scratch = True
+    build_from = BuildFromImageType.SCRATCH
 
     # If the exact image already exists, we do not need to build it
     if runtime_builder.image_exists(hash_image_name, False):
@@ -186,21 +205,32 @@ def build_runtime_image_in_folder(
     # can use it as the base image for the build and just copy source files. This makes the build
     # much faster.
     if runtime_builder.image_exists(lock_image_name):
-        logger.info(f'Build [{hash_image_name}] from [{lock_image_name}]')
-        build_from_scratch = False
+        logger.info(f'Build [{hash_image_name}] from lock image [{lock_image_name}]')
+        build_from = BuildFromImageType.LOCK
         base_image = lock_image_name
+    elif runtime_builder.image_exists(versioned_image_name):
+        logger.info(
+            f'Build [{hash_image_name}] from versioned image [{versioned_image_name}]'
+        )
+        build_from = BuildFromImageType.VERSIONED
+        base_image = versioned_image_name
     else:
         logger.info(f'Build [{hash_image_name}] from scratch')
 
-    prep_build_folder(build_folder, base_image, build_from_scratch, extra_deps)
+    prep_build_folder(build_folder, base_image, build_from, extra_deps)
     if not dry_run:
         _build_sandbox_image(
             build_folder,
             runtime_builder,
             runtime_image_repo,
-            hash_tag,
-            lock_tag,
-            platform,
+            source_tag=source_tag,
+            lock_tag=lock_tag,
+            # Only tag the versioned image if we are building from scratch.
+            # This avoids too much layers when you lay one image on top of another multiple times
+            versioned_tag=versioned_tag
+            if build_from == BuildFromImageType.SCRATCH
+            else None,
+            platform=platform,
         )
 
     return hash_image_name
@@ -209,7 +239,7 @@ def build_runtime_image_in_folder(
 def prep_build_folder(
     build_folder: Path,
     base_image: str,
-    build_from_scratch: bool,
+    build_from: BuildFromImageType,
     extra_deps: str | None,
 ):
     # Copy the source code to directory. It will end up in build_folder/code
@@ -240,7 +270,7 @@ def prep_build_folder(
     # Create a Dockerfile and write it to build_folder
     dockerfile_content = _generate_dockerfile(
         base_image,
-        build_from_scratch=build_from_scratch,
+        build_from=build_from,
         extra_deps=extra_deps,
     )
     with open(Path(build_folder, 'Dockerfile'), 'w') as file:  # type: ignore
@@ -277,6 +307,10 @@ def get_hash_for_lock_files(base_image: str):
     return result
 
 
+def get_tag_for_versioned_image(base_image: str):
+    return base_image.replace('/', '_s_').replace(':', '_t_').lower()[-96:]
+
+
 def get_hash_for_source_files():
     openhands_source_dir = Path(openhands.__file__).parent
     dir_hash = dirhash(
@@ -298,20 +332,19 @@ def _build_sandbox_image(
     build_folder: Path,
     runtime_builder: RuntimeBuilder,
     runtime_image_repo: str,
-    hash_tag: str,
+    source_tag: str,
     lock_tag: str,
+    versioned_tag: str | None,
     platform: str | None = None,
 ):
     """Build and tag the sandbox image. The image will be tagged with all tags that do not yet exist"""
-
     names = [
-        name
-        for name in [
-            f'{runtime_image_repo}:{hash_tag}',
-            f'{runtime_image_repo}:{lock_tag}',
-        ]
-        if not runtime_builder.image_exists(name, False)
+        f'{runtime_image_repo}:{source_tag}',
+        f'{runtime_image_repo}:{lock_tag}',
     ]
+    if versioned_tag is not None:
+        names.append(f'{runtime_image_repo}:{versioned_tag}')
+    names = [name for name in names if not runtime_builder.image_exists(name, False)]
 
     image_name = runtime_builder.build(
         path=str(build_folder), tags=names, platform=platform
@@ -363,8 +396,8 @@ if __name__ == '__main__':
                 platform=args.platform,
             )
 
-            _runtime_image_repo, runtime_image_hash_tag = runtime_image_hash_name.split(
-                ':'
+            _runtime_image_repo, runtime_image_source_tag = (
+                runtime_image_hash_name.split(':')
             )
 
             # Move contents of temp_dir to build_folder
@@ -380,11 +413,11 @@ if __name__ == '__main__':
                 (
                     f'\n'
                     f'DOCKER_IMAGE_TAG={runtime_image_tag}\n'
-                    f'DOCKER_IMAGE_HASH_TAG={runtime_image_hash_tag}\n'
+                    f'DOCKER_IMAGE_SOURCE_TAG={runtime_image_source_tag}\n'
                 )
             )
         logger.info(
-            f'`config.sh` is updated with the image repo[{runtime_image_repo}] and tags [{runtime_image_tag}, {runtime_image_hash_tag}]'
+            f'`config.sh` is updated with the image repo[{runtime_image_repo}] and tags [{runtime_image_tag}, {runtime_image_source_tag}]'
         )
         logger.info(
             f'Dockerfile, source code and config.sh are ready in {build_folder}'

openhands/runtime/utils/runtime_templates/Dockerfile.j2

@@ -4,6 +4,32 @@ FROM {{ base_image }}
 ENV POETRY_VIRTUALENVS_PATH=/openhands/poetry
 ENV MAMBA_ROOT_PREFIX=/openhands/micromamba
 
+{% macro install_dependencies() %}
+# Install all dependencies
+WORKDIR /openhands/code
+RUN \
+    /openhands/micromamba/bin/micromamba config set changeps1 False && \
+    # Configure Poetry and create virtual environment
+    /openhands/micromamba/bin/micromamba run -n openhands poetry config virtualenvs.path /openhands/poetry && \
+    /openhands/micromamba/bin/micromamba run -n openhands poetry env use python3.12 && \
+    # Install project dependencies
+    /openhands/micromamba/bin/micromamba run -n openhands poetry install --only main,runtime --no-interaction --no-root && \
+    # Update and install additional tools
+    apt-get update && \
+    /openhands/micromamba/bin/micromamba run -n openhands poetry run pip install playwright && \
+    /openhands/micromamba/bin/micromamba run -n openhands poetry run playwright install --with-deps chromium && \
+    # Set environment variables
+    echo "OH_INTERPRETER_PATH=$(/openhands/micromamba/bin/micromamba run -n openhands poetry run python -c "import sys; print(sys.executable)")" >> /etc/environment && \
+    # Clear caches
+    /openhands/micromamba/bin/micromamba run -n openhands poetry cache clear --all . && \
+    # Set permissions
+    chmod -R g+rws /openhands/poetry && \
+    mkdir -p /openhands/workspace && chmod -R g+rws,o+rw /openhands/workspace && \
+    # Clean up
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
+    /openhands/micromamba/bin/micromamba clean --all
+{% endmacro %}
+
 {% if build_from_scratch %}
 # ================================================================
 # START: Build Runtime Image from Scratch
@@ -48,29 +74,7 @@ RUN \
     touch /openhands/code/openhands/__init__.py
 COPY ./code/pyproject.toml ./code/poetry.lock /openhands/code/
 
-# Install all dependencies
-WORKDIR /openhands/code
-RUN \
-    /openhands/micromamba/bin/micromamba config set changeps1 False && \
-    # Configure Poetry and create virtual environment
-    /openhands/micromamba/bin/micromamba run -n openhands poetry config virtualenvs.path /openhands/poetry && \
-    /openhands/micromamba/bin/micromamba run -n openhands poetry env use python3.12 && \
-    # Install project dependencies
-    /openhands/micromamba/bin/micromamba run -n openhands poetry install --only main,runtime --no-interaction --no-root && \
-    # Update and install additional tools
-    apt-get update && \
-    /openhands/micromamba/bin/micromamba run -n openhands poetry run pip install playwright && \
-    /openhands/micromamba/bin/micromamba run -n openhands poetry run playwright install --with-deps chromium && \
-    # Set environment variables
-    echo "OH_INTERPRETER_PATH=$(/openhands/micromamba/bin/micromamba run -n openhands poetry run python -c "import sys; print(sys.executable)")" >> /etc/environment && \
-    # Clear caches
-    /openhands/micromamba/bin/micromamba run -n openhands poetry cache clear --all . && \
-    # Set permissions
-    chmod -R g+rws /openhands/poetry && \
-    mkdir -p /openhands/workspace && chmod -R g+rws,o+rw /openhands/workspace && \
-    # Clean up
-    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
-    /openhands/micromamba/bin/micromamba clean --all
+{{ install_dependencies() }}
 
 # ================================================================
 # END: Build Runtime Image from Scratch
@@ -84,5 +88,12 @@ RUN if [ -d /openhands/code/openhands ]; then rm -rf /openhands/code/openhands;
 COPY ./code/pyproject.toml ./code/poetry.lock /openhands/code/
 COPY ./code/openhands /openhands/code/openhands
 
+# ================================================================
+# END: Build from versioned image
+# ================================================================
+{% if build_from_versioned %}
+{{ install_dependencies() }}
+{% endif %}
+
 # Install extra dependencies if specified
 {% if extra_deps %}RUN {{ extra_deps }} {% endif %}

tests/unit/test_runtime_build.py

@@ -16,6 +16,7 @@ from openhands import __version__ as oh_version
 from openhands.core.logger import openhands_logger as logger
 from openhands.runtime.builder.docker import DockerRuntimeBuilder
 from openhands.runtime.utils.runtime_build import (
+    BuildFromImageType,
     _generate_dockerfile,
     build_runtime_image,
     get_hash_for_lock_files,
@@ -83,7 +84,7 @@ def test_prep_build_folder(temp_dir):
         prep_build_folder(
             temp_dir,
             base_image=DEFAULT_BASE_IMAGE,
-            build_from_scratch=True,
+            build_from=BuildFromImageType.SCRATCH,
             extra_deps=None,
         )
 
@@ -130,7 +131,7 @@ def test_generate_dockerfile_build_from_scratch():
     base_image = 'debian:11'
     dockerfile_content = _generate_dockerfile(
         base_image,
-        build_from_scratch=True,
+        build_from=BuildFromImageType.SCRATCH,
     )
     assert base_image in dockerfile_content
     assert 'apt-get update' in dockerfile_content
@@ -146,11 +147,11 @@ def test_generate_dockerfile_build_from_scratch():
     )
 
 
-def test_generate_dockerfile_build_from_existing():
+def test_generate_dockerfile_build_from_lock():
     base_image = 'debian:11'
     dockerfile_content = _generate_dockerfile(
         base_image,
-        build_from_scratch=False,
+        build_from=BuildFromImageType.LOCK,
     )
 
     # These commands SHOULD NOT include in the dockerfile if build_from_scratch is False
@@ -164,6 +165,24 @@ def test_generate_dockerfile_build_from_existing():
     assert 'COPY ./code/openhands /openhands/code/openhands' in dockerfile_content
 
 
+def test_generate_dockerfile_build_from_versioned():
+    base_image = 'debian:11'
+    dockerfile_content = _generate_dockerfile(
+        base_image,
+        build_from=BuildFromImageType.VERSIONED,
+    )
+
+    # these commands should not exist when build from versioned
+    assert 'RUN apt update && apt install -y wget sudo' not in dockerfile_content
+    assert '-c conda-forge' not in dockerfile_content
+    assert 'python=3.12' not in dockerfile_content
+    assert 'https://micro.mamba.pm/install.sh' not in dockerfile_content
+
+    # this SHOULD exist when build from versioned
+    assert 'poetry install' in dockerfile_content
+    assert 'COPY ./code/openhands /openhands/code/openhands' in dockerfile_content
+
+
 def test_get_runtime_image_repo_and_tag_eventstream():
     base_image = 'debian:11'
     img_repo, img_tag = get_runtime_image_repo_and_tag(base_image)
@@ -190,19 +209,22 @@ def test_get_runtime_image_repo_and_tag_eventstream():
 def test_build_runtime_image_from_scratch():
     base_image = 'debian:11'
     mock_lock_hash = MagicMock()
-    mock_lock_hash.return_value = 'mock-lock-hash'
+    mock_lock_hash.return_value = 'mock-lock-tag'
+    mock_versioned_tag = MagicMock()
+    mock_versioned_tag.return_value = 'mock-versioned-tag'
     mock_source_hash = MagicMock()
-    mock_source_hash.return_value = 'mock-source-hash'
+    mock_source_hash.return_value = 'mock-source-tag'
     mock_runtime_builder = MagicMock()
     mock_runtime_builder.image_exists.return_value = False
     mock_runtime_builder.build.return_value = (
-        f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-hash'
+        f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-tag_mock-source-tag'
     )
     mock_prep_build_folder = MagicMock()
     mod = build_runtime_image.__module__
     with (
         patch(f'{mod}.get_hash_for_lock_files', mock_lock_hash),
         patch(f'{mod}.get_hash_for_source_files', mock_source_hash),
+        patch(f'{mod}.get_tag_for_versioned_image', mock_versioned_tag),
         patch(
             f'{build_runtime_image.__module__}.prep_build_folder',
             mock_prep_build_folder,
@@ -212,31 +234,40 @@ def test_build_runtime_image_from_scratch():
         mock_runtime_builder.build.assert_called_once_with(
             path=ANY,
             tags=[
-                f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-hash_mock-source-hash',
-                f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-hash',
+                f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-tag_mock-source-tag',
+                f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-tag',
+                f'{get_runtime_image_repo()}:{OH_VERSION}_mock-versioned-tag',
             ],
             platform=None,
         )
         assert (
             image_name
-            == f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-hash_mock-source-hash'
+            == f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-tag_mock-source-tag'
+        )
+        mock_prep_build_folder.assert_called_once_with(
+            ANY, base_image, BuildFromImageType.SCRATCH, None
         )
-        mock_prep_build_folder.assert_called_once_with(ANY, base_image, True, None)
 
 
 def test_build_runtime_image_exact_hash_exist():
     base_image = 'debian:11'
     mock_lock_hash = MagicMock()
-    mock_lock_hash.return_value = 'mock-lock-hash'
+    mock_lock_hash.return_value = 'mock-lock-tag'
     mock_source_hash = MagicMock()
-    mock_source_hash.return_value = 'mock-source-hash'
+    mock_source_hash.return_value = 'mock-source-tag'
+    mock_versioned_tag = MagicMock()
+    mock_versioned_tag.return_value = 'mock-versioned-tag'
     mock_runtime_builder = MagicMock()
     mock_runtime_builder.image_exists.return_value = True
+    mock_runtime_builder.build.return_value = (
+        f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-tag_mock-source-tag'
+    )
     mock_prep_build_folder = MagicMock()
     mod = build_runtime_image.__module__
     with (
         patch(f'{mod}.get_hash_for_lock_files', mock_lock_hash),
         patch(f'{mod}.get_hash_for_source_files', mock_source_hash),
+        patch(f'{mod}.get_tag_for_versioned_image', mock_versioned_tag),
         patch(
             f'{build_runtime_image.__module__}.prep_build_folder',
             mock_prep_build_folder,
@@ -245,25 +276,45 @@ def test_build_runtime_image_exact_hash_exist():
         image_name = build_runtime_image(base_image, mock_runtime_builder)
         assert (
             image_name
-            == f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-hash_mock-source-hash'
+            == f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-tag_mock-source-tag'
         )
         mock_runtime_builder.build.assert_not_called()
         mock_prep_build_folder.assert_not_called()
 
 
-def test_build_runtime_image_exact_hash_not_exist():
+def test_build_runtime_image_exact_hash_not_exist_and_lock_exist():
     base_image = 'debian:11'
     mock_lock_hash = MagicMock()
-    mock_lock_hash.return_value = 'mock-lock-hash'
+    mock_lock_hash.return_value = 'mock-lock-tag'
     mock_source_hash = MagicMock()
-    mock_source_hash.return_value = 'mock-source-hash'
+    mock_source_hash.return_value = 'mock-source-tag'
+    mock_versioned_tag = MagicMock()
+    mock_versioned_tag.return_value = 'mock-versioned-tag'
     mock_runtime_builder = MagicMock()
-    mock_runtime_builder.image_exists.side_effect = [False, True, False, True]
+
+    def image_exists_side_effect(image_name, *args):
+        if 'mock-lock-tag_mock-source-tag' in image_name:
+            return False
+        elif 'mock-lock-tag' in image_name:
+            return True
+        elif 'mock-versioned-tag' in image_name:
+            # just to test we should never include versioned tag in a non-from-scratch build
+            # in real case it should be True when lock exists
+            return False
+        else:
+            raise ValueError(f'Unexpected image name: {image_name}')
+
+    mock_runtime_builder.image_exists.side_effect = image_exists_side_effect
+    mock_runtime_builder.build.return_value = (
+        f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-tag_mock-source-tag'
+    )
+
     mock_prep_build_folder = MagicMock()
     mod = build_runtime_image.__module__
     with (
         patch(f'{mod}.get_hash_for_lock_files', mock_lock_hash),
         patch(f'{mod}.get_hash_for_source_files', mock_source_hash),
+        patch(f'{mod}.get_tag_for_versioned_image', mock_versioned_tag),
         patch(
             f'{build_runtime_image.__module__}.prep_build_folder',
             mock_prep_build_folder,
@@ -272,11 +323,80 @@ def test_build_runtime_image_exact_hash_not_exist():
         image_name = build_runtime_image(base_image, mock_runtime_builder)
         assert (
             image_name
-            == f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-hash_mock-source-hash'
+            == f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-tag_mock-source-tag'
+        )
+        mock_runtime_builder.build.assert_called_once_with(
+            path=ANY,
+            tags=[
+                f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-tag_mock-source-tag',
+                # lock tag will NOT be included - since it already exists
+                # VERSION tag will NOT be included except from scratch
+            ],
+            platform=None,
+        )
+        mock_prep_build_folder.assert_called_once_with(
+            ANY,
+            f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-tag',
+            BuildFromImageType.LOCK,
+            None,
+        )
+
+
+def test_build_runtime_image_exact_hash_not_exist_and_lock_not_exist_and_versioned_exist():
+    base_image = 'debian:11'
+    mock_lock_hash = MagicMock()
+    mock_lock_hash.return_value = 'mock-lock-tag'
+    mock_source_hash = MagicMock()
+    mock_source_hash.return_value = 'mock-source-tag'
+    mock_versioned_tag = MagicMock()
+    mock_versioned_tag.return_value = 'mock-versioned-tag'
+    mock_runtime_builder = MagicMock()
+
+    def image_exists_side_effect(image_name, *args):
+        if 'mock-lock-tag_mock-source-tag' in image_name:
+            return False
+        elif 'mock-lock-tag' in image_name:
+            return False
+        elif 'mock-versioned-tag' in image_name:
+            return True
+        else:
+            raise ValueError(f'Unexpected image name: {image_name}')
+
+    mock_runtime_builder.image_exists.side_effect = image_exists_side_effect
+    mock_runtime_builder.build.return_value = (
+        f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-tag_mock-source-tag'
+    )
+
+    mock_prep_build_folder = MagicMock()
+    mod = build_runtime_image.__module__
+    with (
+        patch(f'{mod}.get_hash_for_lock_files', mock_lock_hash),
+        patch(f'{mod}.get_hash_for_source_files', mock_source_hash),
+        patch(f'{mod}.get_tag_for_versioned_image', mock_versioned_tag),
+        patch(
+            f'{build_runtime_image.__module__}.prep_build_folder',
+            mock_prep_build_folder,
+        ),
+    ):
+        image_name = build_runtime_image(base_image, mock_runtime_builder)
+        assert (
+            image_name
+            == f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-tag_mock-source-tag'
+        )
+        mock_runtime_builder.build.assert_called_once_with(
+            path=ANY,
+            tags=[
+                f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-tag_mock-source-tag',
+                f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-tag',
+                # VERSION tag will NOT be included except from scratch
+            ],
+            platform=None,
         )
-        mock_runtime_builder.build.assert_called_once()
         mock_prep_build_folder.assert_called_once_with(
-            ANY, f'{get_runtime_image_repo()}:{OH_VERSION}_mock-lock-hash', False, None
+            ANY,
+            f'{get_runtime_image_repo()}:{OH_VERSION}_mock-versioned-tag',
+            BuildFromImageType.VERSIONED,
+            None,
         )