zhang_crawl_bio/utils/proxy_pool.py

import random
from config.settings import PROXIES
import httpx
def get_proxy_pool(proxy_pool_url) -> list:
    res = httpx.get(proxy_pool_url)
    pool_list = res.json().get("proxies")
    if not pool_list:
        res = httpx.get(proxy_pool_url + '?force_refresh=true')
        pool_list = res.json().get("proxies")
    return pool_list
def get_random_proxy(proxy_pool_url: str=None) -> str:
    if proxy_pool_url:
        pool = get_proxy_pool(proxy_pool_url)
        return random.choice(pool)
    if not PROXIES:
        return None
    """Get random proxy from proxy pool"""
    return random.choice(PROXIES)