10个100%有效减少代理IP带宽使用量的方法

1Copy code
2const puppeteer = require('puppeteer');
3
4(async () => {
5  const browser = await puppeteer.launch({ headless: true });
6  const page = await browser.newPage();
7
8  // 屏蔽非必要资源
9  await page.setRequestInterception(true);
10  page.on('request', (req) => {
11    const resourceType = req.resourceType();
12    if (
13      resourceType === 'image' ||
14      resourceType === 'stylesheet' ||
15      resourceType === 'media' ||
16      resourceType === 'font' ||
17      resourceType === 'other'
18    ) {
19      req.abort(); // 屏蔽这些资源
20    } else {
21      req.continue(); // 允许其他资源
22    }
23  });
24
25  await page.goto('https://accounts.google.com/signup', {
26    waitUntil: 'networkidle2',
27  });
28
29  console.log('页面加载完成，资源屏蔽已启用。');
30  await browser.close();
31})();
32
33

1Copy code
2const axios = require('axios');
3
4(async () => {
5  const response = await axios.get('https://example.com/api/data', {
6    headers: {
7      'Accept-Encoding': 'gzip, deflate', // 请求压缩数据
8    },
9    responseType: 'stream', // 以流的方式处理
10  });
11
12  response.data.pipe(process.stdout); // 流式传输压缩数据，避免缓冲区开销
13})();
14
15

1
2const axios = require('axios');
3const Redis = require('ioredis');
4const redis = new Redis();
5
6const CACHE_TTL = 60 * 60; // 缓存1小时
7
8async function fetchWithCache(url) {
9  const cacheKey = `cache:${url}`;
10  const cachedData = await redis.get(cacheKey);
11
12  if (cachedData) {
13    console.log('返回缓存数据...');
14    return JSON.parse(cachedData);
15  }
16
17  console.log('获取新数据...');
18  const response = await axios.get(url);
19  await redis.setex(cacheKey, CACHE_TTL, JSON.stringify(response.data));
20  return response.data;
21}
22
23(async () => {
24  const data = await fetchWithCache('https://example.com/api/data');
25  console.log(data);
26})();
27
28

1
2const response = await axios.get('https://example.com', {
3  headers: {
4    'If-None-Match': '<previous-etag>', // 使用上次请求的ETag
5  },
6});
7

1
2const response = await axios.get('https://example.com/api/users', {
3  params: {
4    page: 1,     // 只抓取第1页
5    limit: 10,   // 限制结果为10条
6  },
7});
8console.log(response.data);
9
10

1
2const http2 = require('http2');
3
4const client = http2.connect('https://example.com');
5const req = client.request({ ':path': '/api/data' });
6
7req.on('data', (chunk) => {
8  console.log(chunk.toString());
9});
10req.end();
11
12

1
2const axios = require('axios');
3
4const proxies = ['http://proxy1:port', 'http://proxy2:port'];
5let currentProxyIndex = 0;
6
7function getNextProxy() {
8  currentProxyIndex = (currentProxyIndex + 1) % proxies.length;
9  return proxies[currentProxyIndex];
10}
11
12async function fetchWithRotation(url) {
13  const proxy = getNextProxy();
14  console.log(`使用代理: ${proxy}`);
15  const response = await axios.get(url, { proxy: { host: proxy } });
16  return response.data;
17}
18
19(async () => {
20  const data = await fetchWithRotation('https://example.com');
21  console.log(data);
22})();
23
24

1
2const processedURLs = new Set();
3
4async function fetchUnique(url) {
5  if (processedURLs.has(url)) {
6    console.log('URL已处理:', url);
7    return;
8  }
9
10  console.log('正在抓取:', url);
11  processedURLs.add(url);
12  const response = await axios.get(url);
13  return response.data;
14}
15
16(async () => {
17  await fetchUnique('https://example.com');
18  await fetchUnique('https://example.com'); // 将被跳过
19})();
20
21