fix(remote runtime): retry on 429 error on remote build & log retries (#4532)

openhands/runtime/builder/remote.py

@@ -7,7 +7,7 @@ import requests
 
 from openhands.core.logger import openhands_logger as logger
 from openhands.runtime.builder import RuntimeBuilder
-from openhands.runtime.utils.request import send_request_with_retry
+from openhands.runtime.utils.request import is_429_error, send_request_with_retry
 from openhands.runtime.utils.shutdown_listener import (
     should_continue,
     sleep_if_should_continue,
@@ -51,6 +51,7 @@ class RemoteRuntimeBuilder(RuntimeBuilder):
             f'{self.api_url}/build',
             files=files,
             timeout=30,
+            retry_fns=[is_429_error],
         )
 
         if response.status_code != 202:

openhands/runtime/utils/request.py

@@ -14,6 +14,7 @@ from tenacity import (
 )
 from urllib3.exceptions import IncompleteRead
 
+from openhands.core.logger import openhands_logger as logger
 from openhands.utils.tenacity_stop import stop_if_should_exit
 
 
@@ -31,6 +32,13 @@ def is_404_error(exception):
     )
 
 
+def is_429_error(exception):
+    return (
+        isinstance(exception, requests.HTTPError)
+        and exception.response.status_code == 429
+    )
+
+
 def is_503_error(exception):
     return (
         isinstance(exception, requests.HTTPError)
@@ -76,6 +84,9 @@ def send_request_with_retry(
         wait=wait_exponential(multiplier=1, min=4, max=20),
         retry=retry_condition,
         reraise=True,
+        before_sleep=lambda retry_state: logger.debug(
+            f'Retrying {method} request to {url} due to {retry_state.outcome.exception()}. Attempt {retry_state.attempt_number}'
+        ),
     )
     def _send_request_with_retry():
         response = session.request(method, url, **kwargs)