request.py 1.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263
  1. from typing import Any, Callable, Type
  2. import requests
  3. from requests.exceptions import ConnectionError, Timeout
  4. from tenacity import (
  5. retry,
  6. retry_if_exception,
  7. retry_if_exception_type,
  8. stop_after_delay,
  9. wait_exponential,
  10. )
  11. def is_server_error(exception):
  12. return (
  13. isinstance(exception, requests.HTTPError)
  14. and exception.response.status_code >= 500
  15. )
  16. def is_404_error(exception):
  17. return (
  18. isinstance(exception, requests.HTTPError)
  19. and exception.response.status_code == 404
  20. )
  21. DEFAULT_RETRY_EXCEPTIONS = [
  22. ConnectionError,
  23. Timeout,
  24. ]
  25. def send_request(
  26. session: requests.Session,
  27. method: str,
  28. url: str,
  29. retry_exceptions: list[Type[Exception]] | None = None,
  30. retry_fns: list[Callable[[Exception], bool]] | None = None,
  31. timeout: int = 120,
  32. **kwargs: Any,
  33. ) -> requests.Response:
  34. exceptions_to_catch = retry_exceptions or DEFAULT_RETRY_EXCEPTIONS
  35. retry_condition = retry_if_exception_type(
  36. tuple(exceptions_to_catch)
  37. ) | retry_if_exception(is_server_error)
  38. if retry_fns is not None:
  39. for fn in retry_fns:
  40. retry_condition |= retry_if_exception(fn)
  41. kwargs["timeout"] = timeout
  42. @retry(
  43. stop=stop_after_delay(timeout),
  44. wait=wait_exponential(multiplier=1, min=4, max=60),
  45. retry=retry_condition,
  46. reraise=True,
  47. )
  48. def _send_request_with_retry():
  49. response = session.request(method, url, **kwargs)
  50. response.raise_for_status()
  51. return response
  52. return _send_request_with_retry()