zhang_crawl_bio/utils/proxy_pool.py

import random
from config.settings import PROXIES
import httpx
def get_proxy_pool(proxy_pool_url) -> list:
    res = httpx.get(proxy_pool_url)
    return res.json().get("proxies")

def get_random_proxy(proxy_pool_url: str=None) -> str:
    if proxy_pool_url:
        pool = get_proxy_pool(proxy_pool_url)
        return random.choice(pool)
    if not PROXIES:
        return None
    """Get random proxy from proxy pool"""
    return random.choice(PROXIES)