深入解析：如何使用 cURL 获取 YouTube 网页内容的专业指南

1# 基础请求
2curl "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
3
4# 保存到文件
5curl -o youtube_raw.html "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
6
7

1curl -H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" \
2     "https://www.youtube.com/watch?v=VIDEO_ID"
3
4

1#!/bin/bash
2
3URL="https://www.youtube.com/watch?v=dQw4w9WgXcQ"
4OUTPUT_FILE="youtube_response.html"
5
6curl -L \
7  -H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" \
8  -H "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8" \
9  -H "Accept-Language: en-US,en;q=0.9" \
10  -H "Accept-Encoding: gzip, deflate, br" \
11  -H "Referer: https://www.youtube.com/" \
12  -H "DNT: 1" \
13  -H "Connection: keep-alive" \
14  -H "Upgrade-Insecure-Requests: 1" \
15  -H "Sec-Fetch-Dest: document" \
16  -H "Sec-Fetch-Mode: navigate" \
17  -H "Sec-Fetch-Site: same-origin" \
18  -H "Sec-Fetch-User: ?1" \
19  -H "Cache-Control: max-age=0" \
20  -H "sec-ch-ua: '\"Not_A Brand\";v=\"8\", \"Chromium\";v=\"120\", \"Google Chrome\";v=\"120\"'" \
21  -H "sec-ch-ua-mobile: ?0" \
22  -H "sec-ch-ua-platform: '\"Windows\"'" \
23  --compressed \
24  -o "$OUTPUT_FILE" \
25  "$URL"
26
27

1# 保存 cookie
2curl -c cookies.txt "https://www.youtube.com"
3
4# 使用保存的 cookie
5curl -b cookies.txt -c cookies.txt "https://www.youtube.com/watch?v=VIDEO_ID"
6
7

1# 1. 查找内联脚本数据
2curl -s "$URL" | grep -o '"videoDetails":{[^}]*}' | head -5
3
4# 2. 使用开发者工具识别数据端点
5# 通过浏览器开发者工具 > 网络标签，查找 XHR/Fetch 请求
6
7

1# 添加延迟避免频率限制
2curl --max-time 30 --retry 2 "$URL"
3
4# 使用代理轮换
5curl --proxy http://proxy-server:port "$URL"
6
7# 限制请求频率（使用 shell 脚本控制）
8for video_id in "${video_ids[@]}"; do
9    curl "https://www.youtube.com/watch?v=$video_id" -o "${video_id}.html"
10    sleep $((RANDOM % 5 + 2))  # 随机延迟 2-7 秒
11done
12
13

1# 尝试获取初始数据包
2INITIAL_DATA=$(curl -s "$URL" | grep -o 'var ytInitialData = {.*};' | sed 's/var ytInitialData = //' | sed 's/;$//')
3
4echo "$INITIAL_DATA" | jq .  # 使用 jq 解析 JSON
5
6

1# 使用 API 密钥请求视频数据
2API_KEY="YOUR_API_KEY"
3VIDEO_ID="dQw4w9WgXcQ"
4
5curl "https://www.googleapis.com/youtube/v3/videos?part=snippet,contentDetails,statistics&id=$VIDEO_ID&key=$API_KEY"
6
7# 响应为结构化 JSON
8
9

1# 获取视频元数据（JSON 格式）
2yt-dlp -j "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
3
4# 获取特定格式信息
5yt-dlp -F "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
6
7# 提取视频描述
8yt-dlp --get-description "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
9
10

1# 使用 Puppeteer 的 Node.js 脚本
2node -e "
3const puppeteer = require('puppeteer');
4
5(async () => {
6    const browser = await puppeteer.launch();
7    const page = await browser.newPage();
8    await page.goto('https://www.youtube.com/watch?v=dQw4w9WgXcQ', {waitUntil: 'networkidle2'});
9    const content = await page.content();
10    console.log(content);
11    await browser.close();
12})();
13"
14
15

1#!/bin/bash
2# extract_youtube_metadata.sh
3
4VIDEO_ID="$1"
5TEMP_FILE="/tmp/youtube_$$.html"
6
7# 获取页面内容
8curl -s \
9  -H "User-Agent: $(shuf -n 1 user_agents.txt)" \
10  -H "Accept-Language: en-US,en;q=0.9" \
11  "https://www.youtube.com/watch?v=$VIDEO_ID" > "$TEMP_FILE"
12
13# 提取标题
14TITLE=$(grep -o '<title>[^<]*</title>' "$TEMP_FILE" | sed 's/<title>//' | sed 's/<\/title>//' | sed 's/ - YouTube$//')
15
16# 查找初始数据
17INITIAL_DATA=$(grep -o 'var ytInitialData = {.*};' "$TEMP_FILE" | sed 's/var ytInitialData = //' | sed 's/;$//' 2>/dev/null)
18
19if [ -n "$INITIAL_DATA" ]; then
20    # 使用 jq 解析 JSON（需要安装 jq）
21    VIEWS=$(echo "$INITIAL_DATA" | jq -r '.contents.twoColumnWatchNextResults.results.results.contents[1].videoSecondaryInfoRenderer.metadataRowContainer.metadataRowContainerRenderer.rows[0].metadataRowRenderer.contents[0].simpleText' 2>/dev/null)
22    
23    echo "Video ID: $VIDEO_ID"
24    echo "Title: $TITLE"
25    echo "Views: $VIEWS"
26fi
27
28rm "$TEMP_FILE"
29
30

1#!/bin/bash
2# monitor_youtube_video.sh
3
4VIDEO_ID="dQw4w9WgXcQ"
5CHECK_INTERVAL=300  # 5分钟
6PREVIOUS_VIEWS=""
7
8while true; do
9    CURRENT_DATA=$(curl -s "https://www.googleapis.com/youtube/v3/videos?part=statistics&id=$VIDEO_ID&key=$API_KEY")
10    CURRENT_VIEWS=$(echo "$CURRENT_DATA" | jq -r '.items[0].statistics.viewCount')
11    
12    if [ "$CURRENT_VIEWS" != "$PREVIOUS_VIEWS" ]; then
13        echo "$(date): Views changed from $PREVIOUS_VIEWS to $CURRENT_VIEWS"
14        PREVIOUS_VIEWS="$CURRENT_VIEWS"
15    fi
16    
17    sleep $CHECK_INTERVAL
18done
19
20