Fix issue #4480: '[Bug]: Being blocked by cloudflare results in futile retries (#4482)

Co-authored-by: openhands <openhands@all-hands.dev>

openhands/core/exceptions.py

@@ -84,3 +84,9 @@ class OperationCancelled(Exception):
 
     def __init__(self, message='Operation was cancelled'):
         super().__init__(message)
+
+
+class CloudFlareBlockageError(Exception):
+    """Exception raised when a request is blocked by CloudFlare."""
+
+    pass

openhands/llm/llm.py

@@ -21,6 +21,7 @@ from litellm.exceptions import (
 )
 from litellm.types.utils import CostPerToken, ModelResponse, Usage
 
+from openhands.core.exceptions import CloudFlareBlockageError
 from openhands.core.logger import openhands_logger as logger
 from openhands.core.message import Message
 from openhands.core.metrics import Metrics
@@ -187,29 +188,36 @@ class LLM(RetryMixin, DebugMixin):
                         'anthropic-beta': 'prompt-caching-2024-07-31',
                     }
 
-            # we don't support streaming here, thus we get a ModelResponse
-            resp: ModelResponse = completion_unwrapped(*args, **kwargs)
-
-            # log for evals or other scripts that need the raw completion
-            if self.config.log_completions:
-                self.llm_completions.append(
-                    {
-                        'messages': messages,
-                        'response': resp,
-                        'timestamp': time.time(),
-                        'cost': self._completion_cost(resp),
-                    }
-                )
-
-            message_back: str = resp['choices'][0]['message']['content']
-
-            # log the LLM response
-            self.log_response(message_back)
-
-            # post-process the response
-            self._post_completion(resp)
-
-            return resp
+            try:
+                # we don't support streaming here, thus we get a ModelResponse
+                resp: ModelResponse = completion_unwrapped(*args, **kwargs)
+
+                # log for evals or other scripts that need the raw completion
+                if self.config.log_completions:
+                    self.llm_completions.append(
+                        {
+                            'messages': messages,
+                            'response': resp,
+                            'timestamp': time.time(),
+                            'cost': self._completion_cost(resp),
+                        }
+                    )
+
+                message_back: str = resp['choices'][0]['message']['content']
+
+                # log the LLM response
+                self.log_response(message_back)
+
+                # post-process the response
+                self._post_completion(resp)
+
+                return resp
+            except APIError as e:
+                if 'Attention Required! | Cloudflare' in str(e):
+                    raise CloudFlareBlockageError(
+                        'Request blocked by CloudFlare'
+                    ) from e
+                raise
 
         self._completion = wrapper
 

tests/unit/test_llm.py

@@ -332,3 +332,24 @@ def test_completion_with_two_positional_args(mock_litellm_completion, default_co
     assert (
         len(call_args) == 0
     )  # No positional args should be passed to litellm_completion here
+
+
+@patch('openhands.llm.llm.litellm_completion')
+def test_llm_cloudflare_blockage(mock_litellm_completion, default_config):
+    from litellm.exceptions import APIError
+
+    from openhands.core.exceptions import CloudFlareBlockageError
+
+    llm = LLM(default_config)
+    mock_litellm_completion.side_effect = APIError(
+        message='Attention Required! | Cloudflare',
+        llm_provider='test_provider',
+        model='test_model',
+        status_code=403,
+    )
+
+    with pytest.raises(CloudFlareBlockageError, match='Request blocked by CloudFlare'):
+        llm.completion(messages=[{'role': 'user', 'content': 'Hello'}])
+
+    # Ensure the completion was called
+    mock_litellm_completion.assert_called_once()