怎么实现爬虫自动切换代理IP？

1import requests
2from itertools import cycle
3
4# 获取代理列表（示例代理列表）
5proxies = [
6    "http://proxy.momoproxy.com:8080",
7    "http://proxy.momoproxy.com:8080",
8    # 更多代理...
9]
10
11# 创建一个代理池
12proxy_pool = cycle(proxies)
13
14def get_proxy():
15    return next(proxy_pool)
16
17def check_proxy(proxy):
18    try:
19        response = requests.get("http://httpbin.org/ip", proxies={"http": proxy, "https": proxy}, timeout=5)
20        if response.status_code == 200:
21            print(f"Proxy {proxy} is working")
22            return True
23    except:
24        print(f"Proxy {proxy} failed")
25    return False
26
27# 在爬虫中使用代理
28def fetch_url(url):
29    while True:
30        proxy = get_proxy()
31        try:
32            response = requests.get(url, proxies={"http": proxy, "https": proxy}, timeout=5)
33            if response.status_code == 200:
34                print(f"Successfully fetched {url} with proxy {proxy}")
35                return response.text
36            else:
37                print(f"Failed to fetch {url} with proxy {proxy}, status code: {response.status_code}")
38        except Exception as e:
39            print(f"Error fetching {url} with proxy {proxy}: {e}")
40            print("Switching proxy...")
41
42# 示例爬取
43url = "http://example.com"
44content = fetch_url(url)
45print(content)
46
47