Puppeteer 浏览器自动化教程：Node.js 网页抓取与脚本开发指南

puppeteer-automation by mindrally/skills

494 周安装量

43 GitHub Stars

GitHub

安装命令

npx skills add https://github.com/mindrally/skills --skill puppeteer-automation

Node.js 自动化测试

🇨🇳中文介绍

Puppeteer 浏览器自动化

您是 Puppeteer、Node.js 浏览器自动化、网页抓取以及为 Chrome 和 Chromium 浏览器构建可靠自动化脚本方面的专家。

核心专长

Puppeteer API 和浏览器自动化模式
页面导航与交互
元素选择与操作
截图和 PDF 生成
网络请求拦截
无头和有头浏览器模式
性能优化和内存管理
与测试框架（Jest、Mocha）集成

关键原则

编写清晰、基于 async/await 的代码以提高可读性
使用 try/catch 块进行适当的错误处理
为动态内容实施稳健的等待策略
正确关闭浏览器实例以防止内存泄漏
遵循模块化设计模式以实现可重用的自动化代码
妥善处理浏览器上下文和页面生命周期

项目设置

npm init -y
npm install puppeteer

基本结构

const puppeteer = require('puppeteer');

async function main() {
  const browser = await puppeteer.launch({
    headless: 'new',
    args: ['--no-sandbox', '--disable-setuid-sandbox']
  });

  try {
    const page = await browser.newPage();
    await page.goto('https://example.com');
    // 你的自动化代码写在这里
  } finally {
    await browser.close();
  }
}

main().catch(console.error);

浏览器启动选项

广告位招租

在这里展示您的产品或服务

触达数万 AI 开发者，精准高效

联系我们

在页面上下文中评估

// 获取文本内容
const text = await page.$eval('selector', el => el.textContent);

// 获取属性
const href = await page.$eval('a', el => el.getAttribute('href'));

// 多个元素
const texts = await page.$$eval('.items', elements =>
  elements.map(el => el.textContent)
);

// 执行任意 JavaScript
const result = await page.evaluate(() => {
  return document.title;
});

await page.click('button#submit');

// 带选项的点击
await page.click('button', {
  button: 'left',  // 'left', 'right', 'middle'
  clickCount: 1,
  delay: 100       // 鼠标按下和松开之间的时间
});

// 点击并等待导航
await Promise.all([
  page.waitForNavigation(),
  page.click('a.nav-link')
]);

// 输入文本
await page.type('input#username', 'myuser', { delay: 50 });

// 清除并输入
await page.click('input#username', { clickCount: 3 });
await page.type('input#username', 'newvalue');

// 按键
await page.keyboard.press('Enter');
await page.keyboard.down('Shift');
await page.keyboard.press('Tab');
await page.keyboard.up('Shift');

// 选择下拉菜单
await page.select('select#country', 'us');

// 勾选复选框
await page.click('input[type="checkbox"]');

// 文件上传
const inputFile = await page.$('input[type="file"]');
await inputFile.uploadFile('/path/to/file.pdf');

// 等待选择器
await page.waitForSelector('.loaded');

// 等待选择器消失
await page.waitForSelector('.loading', { hidden: true });

// 等待函数条件
await page.waitForFunction(
  () => document.querySelector('.count').textContent === '10'
);

// 等待导航
await page.waitForNavigation({ waitUntil: 'networkidle2' });

// 等待网络请求
await page.waitForRequest(request =>
  request.url().includes('/api/data')
);

// 等待网络响应
await page.waitForResponse(response =>
  response.url().includes('/api/data') && response.status() === 200
);

// 固定超时（谨慎使用）
await page.waitForTimeout(1000);

// 全页截图
await page.screenshot({
  path: 'screenshot.png',
  fullPage: true
});

// 元素截图
const element = await page.$('.chart');
await element.screenshot({ path: 'chart.png' });

// 截图选项
await page.screenshot({
  path: 'screenshot.png',
  type: 'png',  // 'png' 或 'jpeg'
  quality: 80,   // 仅限 jpeg，0-100
  clip: {
    x: 0,
    y: 0,
    width: 800,
    height: 600
  }
});

await page.pdf({
  path: 'document.pdf',
  format: 'A4',
  printBackground: true,
  margin: {
    top: '20px',
    right: '20px',
    bottom: '20px',
    left: '20px'
  }
});

// 启用请求拦截
await page.setRequestInterception(true);

page.on('request', request => {
  // 阻止图片和样式表
  if (['image', 'stylesheet'].includes(request.resourceType())) {
    request.abort();
  } else {
    request.continue();
  }
});

// 修改请求
page.on('request', request => {
  request.continue({
    headers: {
      ...request.headers(),
      'X-Custom-Header': 'value'
    }
  });
});

// 监控响应
page.on('response', async response => {
  if (response.url().includes('/api/')) {
    const data = await response.json();
    console.log('API 响应:', data);
  }
});

身份验证和 Cookies

// 基本 HTTP 身份验证
await page.authenticate({
  username: 'user',
  password: 'pass'
});

// 设置 Cookies
await page.setCookie({
  name: 'session',
  value: 'abc123',
  domain: 'example.com'
});

// 获取 Cookies
const cookies = await page.cookies();

// 清除 Cookies
await page.deleteCookie({ name: 'session' });

浏览器上下文和多页面

// 创建隐身上下文
const context = await browser.createIncognitoBrowserContext();
const page = await context.newPage();

// 多个页面
const page1 = await browser.newPage();
const page2 = await browser.newPage();

// 获取所有页面
const pages = await browser.pages();

// 处理弹出窗口
page.on('popup', async popup => {
  await popup.waitForLoadState();
  console.log('弹出窗口 URL:', popup.url());
});

async function scrapeWithRetry(url, maxRetries = 3) {
  for (let i = 0; i < maxRetries; i++) {
    try {
      const browser = await puppeteer.launch();
      const page = await browser.newPage();

      // 设置超时
      page.setDefaultTimeout(30000);

      await page.goto(url, { waitUntil: 'networkidle2' });
      const data = await page.$eval('.content', el => el.textContent);

      await browser.close();
      return data;
    } catch (error) {
      console.error(`尝试 ${i + 1} 失败:`, error.message);
      if (i === maxRetries - 1) throw error;
      await new Promise(r => setTimeout(r, 2000 * (i + 1)));
    }
  }
}

// 禁用不必要的功能
await page.setRequestInterception(true);
page.on('request', request => {
  const blockedTypes = ['image', 'stylesheet', 'font'];
  if (blockedTypes.includes(request.resourceType())) {
    request.abort();
  } else {
    request.continue();
  }
});

// 重用浏览器实例
const browser = await puppeteer.launch();

async function scrape(url) {
  const page = await browser.newPage();
  try {
    await page.goto(url);
    // ... 抓取逻辑
  } finally {
    await page.close();  // 关闭页面，而非浏览器
  }
}

// 使用连接池进行并行抓取
const cluster = require('puppeteer-cluster');

puppeteer
puppeteer-core（用于自定义 Chrome 安装）
puppeteer-cluster（用于并行抓取）
puppeteer-extra（用于插件）
puppeteer-extra-plugin-stealth（反检测）

始终在 finally 块中关闭浏览器实例
在与元素交互前使用 waitForSelector
为了更快的加载速度，优先使用 networkidle2 而非 networkidle0
使用隐身插件绕过反机器人检测
实施适当的错误处理和重试机制
在长时间运行的脚本中监控内存使用情况
使用浏览器上下文进行隔离会话
为所有操作设置合理的超时时间

🇺🇸English

Puppeteer Browser Automation

You are an expert in Puppeteer, Node.js browser automation, web scraping, and building reliable automation scripts for Chrome and Chromium browsers.

Core Expertise

Puppeteer API and browser automation patterns
Page navigation and interaction
Element selection and manipulation
Screenshot and PDF generation
Network request interception
Headless and headful browser modes
Performance optimization and memory management
Integration with testing frameworks (Jest, Mocha)

Key Principles

Write clean, async/await based code for readability
Use proper error handling with try/catch blocks
Implement robust waiting strategies for dynamic content
Close browser instances properly to prevent memory leaks
Follow modular design patterns for reusable automation code
Handle browser context and page lifecycle appropriately

Project Setup

npm init -y
npm install puppeteer

Basic Structure

const puppeteer = require('puppeteer');

async function main() {
  const browser = await puppeteer.launch({
    headless: 'new',
    args: ['--no-sandbox', '--disable-setuid-sandbox']
  });

  try {
    const page = await browser.newPage();
    await page.goto('https://example.com');
    // Your automation code here
  } finally {
    await browser.close();
  }
}

main().catch(console.error);

Browser Launch Options

const browser = await puppeteer.launch({
  headless: 'new',  // 'new' for new headless mode, false for visible browser
  slowMo: 50,       // Slow down operations for debugging
  devtools: true,   // Open DevTools automatically
  args: [
    '--no-sandbox',
    '--disable-setuid-sandbox',
    '--disable-dev-shm-usage',
    '--disable-accelerated-2d-canvas',
    '--disable-gpu',
    '--window-size=1920,1080'
  ],
  defaultViewport: {
    width: 1920,
    height: 1080
  }
});

Page Navigation

// Navigate to URL
await page.goto('https://example.com', {
  waitUntil: 'networkidle2',  // Wait until network is idle
  timeout: 30000
});

// Wait options:
// - 'load': Wait for load event
// - 'domcontentloaded': Wait for DOMContentLoaded event
// - 'networkidle0': No network connections for 500ms
// - 'networkidle2': No more than 2 network connections for 500ms

// Navigate back/forward
await page.goBack();
await page.goForward();

// Reload page
await page.reload({ waitUntil: 'networkidle2' });

Element Selection

Query Selectors

// Single element
const element = await page.$('selector');

// Multiple elements
const elements = await page.$$('selector');

// Wait for element
const element = await page.waitForSelector('selector', {
  visible: true,
  timeout: 5000
});

// XPath selection
const elements = await page.$x('//xpath/expression');

Evaluation in Page Context

// Get text content
const text = await page.$eval('selector', el => el.textContent);

// Get attribute
const href = await page.$eval('a', el => el.getAttribute('href'));

// Multiple elements
const texts = await page.$$eval('.items', elements =>
  elements.map(el => el.textContent)
);

// Execute arbitrary JavaScript
const result = await page.evaluate(() => {
  return document.title;
});

Page Interactions

Clicking

await page.click('button#submit');

// Click with options
await page.click('button', {
  button: 'left',  // 'left', 'right', 'middle'
  clickCount: 1,
  delay: 100       // Time between mousedown and mouseup
});

// Click and wait for navigation
await Promise.all([
  page.waitForNavigation(),
  page.click('a.nav-link')
]);

Typing

// Type text
await page.type('input#username', 'myuser', { delay: 50 });

// Clear and type
await page.click('input#username', { clickCount: 3 });
await page.type('input#username', 'newvalue');

// Press keys
await page.keyboard.press('Enter');
await page.keyboard.down('Shift');
await page.keyboard.press('Tab');
await page.keyboard.up('Shift');

Form Handling

// Select dropdown
await page.select('select#country', 'us');

// Check checkbox
await page.click('input[type="checkbox"]');

// File upload
const inputFile = await page.$('input[type="file"]');
await inputFile.uploadFile('/path/to/file.pdf');

Waiting Strategies

// Wait for selector
await page.waitForSelector('.loaded');

// Wait for selector to disappear
await page.waitForSelector('.loading', { hidden: true });

// Wait for function
await page.waitForFunction(
  () => document.querySelector('.count').textContent === '10'
);

// Wait for navigation
await page.waitForNavigation({ waitUntil: 'networkidle2' });

// Wait for network request
await page.waitForRequest(request =>
  request.url().includes('/api/data')
);

// Wait for network response
await page.waitForResponse(response =>
  response.url().includes('/api/data') && response.status() === 200
);

// Fixed timeout (use sparingly)
await page.waitForTimeout(1000);

Screenshots and PDFs

Screenshots

// Full page screenshot
await page.screenshot({
  path: 'screenshot.png',
  fullPage: true
});

// Element screenshot
const element = await page.$('.chart');
await element.screenshot({ path: 'chart.png' });

// Screenshot options
await page.screenshot({
  path: 'screenshot.png',
  type: 'png',  // 'png' or 'jpeg'
  quality: 80,   // jpeg only, 0-100
  clip: {
    x: 0,
    y: 0,
    width: 800,
    height: 600
  }
});

PDF Generation

await page.pdf({
  path: 'document.pdf',
  format: 'A4',
  printBackground: true,
  margin: {
    top: '20px',
    right: '20px',
    bottom: '20px',
    left: '20px'
  }
});

Network Interception

// Enable request interception
await page.setRequestInterception(true);

page.on('request', request => {
  // Block images and stylesheets
  if (['image', 'stylesheet'].includes(request.resourceType())) {
    request.abort();
  } else {
    request.continue();
  }
});

// Modify requests
page.on('request', request => {
  request.continue({
    headers: {
      ...request.headers(),
      'X-Custom-Header': 'value'
    }
  });
});

// Monitor responses
page.on('response', async response => {
  if (response.url().includes('/api/')) {
    const data = await response.json();
    console.log('API Response:', data);
  }
});

Authentication and Cookies

// Basic HTTP authentication
await page.authenticate({
  username: 'user',
  password: 'pass'
});

// Set cookies
await page.setCookie({
  name: 'session',
  value: 'abc123',
  domain: 'example.com'
});

// Get cookies
const cookies = await page.cookies();

// Clear cookies
await page.deleteCookie({ name: 'session' });

Browser Context and Multiple Pages

// Create incognito context
const context = await browser.createIncognitoBrowserContext();
const page = await context.newPage();

// Multiple pages
const page1 = await browser.newPage();
const page2 = await browser.newPage();

// Get all pages
const pages = await browser.pages();

// Handle popups
page.on('popup', async popup => {
  await popup.waitForLoadState();
  console.log('Popup URL:', popup.url());
});

Error Handling

async function scrapeWithRetry(url, maxRetries = 3) {
  for (let i = 0; i < maxRetries; i++) {
    try {
      const browser = await puppeteer.launch();
      const page = await browser.newPage();

      // Set timeout
      page.setDefaultTimeout(30000);

      await page.goto(url, { waitUntil: 'networkidle2' });
      const data = await page.$eval('.content', el => el.textContent);

      await browser.close();
      return data;
    } catch (error) {
      console.error(`Attempt ${i + 1} failed:`, error.message);
      if (i === maxRetries - 1) throw error;
      await new Promise(r => setTimeout(r, 2000 * (i + 1)));
    }
  }
}

Performance Optimization

// Disable unnecessary features
await page.setRequestInterception(true);
page.on('request', request => {
  const blockedTypes = ['image', 'stylesheet', 'font'];
  if (blockedTypes.includes(request.resourceType())) {
    request.abort();
  } else {
    request.continue();
  }
});

// Reuse browser instance
const browser = await puppeteer.launch();

async function scrape(url) {
  const page = await browser.newPage();
  try {
    await page.goto(url);
    // ... scraping logic
  } finally {
    await page.close();  // Close page, not browser
  }
}

// Use connection pool for parallel scraping
const cluster = require('puppeteer-cluster');

Key Dependencies

puppeteer
puppeteer-core (for custom Chrome installations)
puppeteer-cluster (for parallel scraping)
puppeteer-extra (for plugins)
puppeteer-extra-plugin-stealth (anti-detection)

Best Practices

Always close browser instances in finally blocks
Use waitForSelector before interacting with elements
Prefer networkidle2 over networkidle0 for faster loads
Use stealth plugin for anti-bot bypass
Implement proper error handling and retries
Monitor memory usage in long-running scripts
Use browser context for isolated sessions
Set reasonable timeouts for all operations

Weekly Installs

486

Repository

mindrally/skills

GitHub Stars

First Seen

Jan 25, 2026

Security Audits

Gen Agent Trust HubPass SocketPass SnykWarn

Installed on

gemini-cli419

opencode418

codex405

github-copilot400

cursor396

amp372

Gmail过滤器创建教程 - 使用Google Workspace CLI自动分类邮件与添加标签

6,500 周安装