request.py 1.5 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162
  1. from typing import Any, Callable, Type
  2. import requests
  3. from requests.exceptions import ConnectionError, Timeout
  4. from tenacity import (
  5. retry,
  6. retry_if_exception,
  7. retry_if_exception_type,
  8. stop_after_attempt,
  9. wait_exponential,
  10. )
  11. def is_server_error(exception):
  12. return (
  13. isinstance(exception, requests.HTTPError)
  14. and exception.response.status_code >= 500
  15. )
  16. def is_404_error(exception):
  17. return (
  18. isinstance(exception, requests.HTTPError)
  19. and exception.response.status_code == 404
  20. )
  21. DEFAULT_RETRY_EXCEPTIONS = [
  22. ConnectionError,
  23. Timeout,
  24. ]
  25. def send_request(
  26. session: requests.Session,
  27. method: str,
  28. url: str,
  29. retry_exceptions: list[Type[Exception]] | None = None,
  30. retry_fns: list[Callable[[Exception], bool]] | None = None,
  31. n_attempts: int = 15,
  32. **kwargs: Any,
  33. ) -> requests.Response:
  34. exceptions_to_catch = retry_exceptions or DEFAULT_RETRY_EXCEPTIONS
  35. retry_condition = retry_if_exception_type(
  36. tuple(exceptions_to_catch)
  37. ) | retry_if_exception(is_server_error)
  38. if retry_fns is not None:
  39. for fn in retry_fns:
  40. retry_condition |= retry_if_exception(fn)
  41. @retry(
  42. stop=stop_after_attempt(n_attempts),
  43. wait=wait_exponential(multiplier=1, min=4, max=60),
  44. retry=retry_condition,
  45. reraise=True,
  46. )
  47. def _send_request_with_retry():
  48. response = session.request(method, url, **kwargs)
  49. response.raise_for_status()
  50. return response
  51. return _send_request_with_retry()