Rustでスクレイピング入門｜reqwest・scraper・headless_chromeの実践コード付き

curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh

cargo new rust-scraper-demo
cd rust-scraper-demo

[package]
name = "rust-scraper-demo"
version = "0.1.0"
edition = "2021"

[dependencies]
reqwest = { version = "0.13", features = ["blocking"] }
scraper = "0.25"

use reqwest::blocking::get;
use scraper::{Html, Selector};

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let url = "https://www.rust-lang.org/";
    let response = get(url)?;
    let body = response.text()?;

    let document = Html::parse_document(&body);
    let selector = Selector::parse("title").unwrap();

    for element in document.select(&selector) {
        let title = element.text().collect::<String>();
        println!("タイトル: {}", title);
    }

    Ok(())
}

use reqwest::blocking::get;
use scraper::{Html, Selector};

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let url = "https://www.rust-lang.org/";
    let body = get(url)?.text()?;
    let document = Html::parse_document(&body);

    let link_selector = Selector::parse("a[href]").unwrap();

    for element in document.select(&link_selector) {
        let href = element.value().attr("href").unwrap_or("");
        let text: String = element.text().collect();
        if !text.trim().is_empty() {
            println!("{} -> {}", text.trim(), href);
        }
    }

    Ok(())
}

[dependencies]
reqwest = "0.13"
scraper = "0.25"
tokio = { version = "1", features = ["full"] }

use reqwest::Client;
use scraper::{Html, Selector};

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    let client = Client::new();
    let urls = vec![
        "https://www.rust-lang.org/",
        "https://crates.io/",
        "https://doc.rust-lang.org/book/",
    ];

    let mut handles = vec![];

    for url in urls {
        let client = client.clone();
        let url = url.to_string();
        let handle = tokio::spawn(async move {
            let resp = client.get(&url).send().await;
            match resp {
                Ok(r) => {
                    let body = r.text().await.unwrap_or_default();
                    let doc = Html::parse_document(&body);
                    let sel = Selector::parse("title").unwrap();
                    let title: String = doc
                        .select(&sel)
                        .next()
                        .map(|e| e.text().collect())
                        .unwrap_or_default();
                    println!("[{}] {}", url, title);
                }
                Err(e) => eprintln!("[{}] エラー: {}", url, e),
            }
        });
        handles.push(handle);
    }

    for handle in handles {
        handle.await?;
    }

    Ok(())
}

use std::time::Duration;
use tokio::time::sleep;

async fn fetch_with_delay(client: &Client, url: &str) -> Result<String, reqwest::Error> {
    sleep(Duration::from_millis(500)).await;
    let resp = client.get(url).send().await?;
    resp.text().await
}

use reqwest::header::{HeaderMap, HeaderValue, USER_AGENT, ACCEPT_LANGUAGE};

fn build_headers() -> HeaderMap {
    let mut headers = HeaderMap::new();
    headers.insert(USER_AGENT, HeaderValue::from_static("Mozilla/5.0 (compatible; MyBot/1.0)"));
    headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static("ja,en;q=0.9"));
    headers
}

use reqwest::blocking::Client;
use std::collections::HashMap;

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let client = Client::builder()
        .cookie_store(true)
        .build()?;

    // ログインリクエスト
    let mut params = HashMap::new();
    params.insert("username", "user");
    params.insert("password", "pass");

    client.post("https://example.com/login")
        .form(&params)
        .send()?;

    // ログイン後のページを取得（Cookieが自動送信される）
    let protected = client.get("https://example.com/dashboard")
        .send()?
        .text()?;

    println!("取得したページ長: {} bytes", protected.len());
    Ok(())
}

use reqwest::blocking::Client;

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let client = Client::new();

    // フォームデータの送信
    let params = [("q", "Rust scraping"), ("lang", "ja")];
    let resp = client.post("https://httpbin.org/post")
        .form(&params)
        .send()?;

    println!("ステータス: {}", resp.status());
    println!("レスポンス: {}", resp.text()?);

    Ok(())
}

[dependencies]
headless_chrome = "1.0"

use headless_chrome::Browser;

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let browser = Browser::default()?;
    let tab = browser.new_tab()?;

    tab.navigate_to("https://www.rust-lang.org/")?;
    tab.wait_until_navigated()?;

    // JavaScript実行後のHTMLを取得
    let html = tab.get_content()?;
    println!("HTML長: {} bytes", html.len());

    // 特定の要素のテキストを取得
    let element = tab.wait_for_element("h1")?;
    let text = element.get_inner_text()?;
    println!("h1テキスト: {}", text);

    Ok(())
}

use headless_chrome::Browser;
use headless_chrome::protocol::cdp::Page::CaptureScreenshotFormatOption;
use std::fs;

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let browser = Browser::default()?;
    let tab = browser.new_tab()?;

    tab.navigate_to("https://www.rust-lang.org/")?;
    tab.wait_until_navigated()?;

    let screenshot = tab.capture_screenshot(
        CaptureScreenshotFormatOption::Png,
        None,
        None,
        true,
    )?;

    fs::write("screenshot.png", &screenshot)?;
    println!("スクリーンショットを保存しました");

    Ok(())
}

use headless_chrome::Browser;

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let browser = Browser::default()?;
    let tab = browser.new_tab()?;

    tab.navigate_to("https://example.com/login")?;
    tab.wait_until_navigated()?;

    // テキスト入力
    tab.wait_for_element("input[name='username']")?
        .click()?;
    tab.type_str("myuser")?;

    tab.wait_for_element("input[name='password']")?
        .click()?;
    tab.type_str("mypass")?;

    // ボタンクリック
    tab.wait_for_element("button[type='submit']")?
        .click()?;

    tab.wait_until_navigated()?;
    let content = tab.get_content()?;
    println!("ログイン後のHTML長: {} bytes", content.len());

    Ok(())
}

use reqwest::blocking::Client;
use std::time::Duration;
use std::thread;

fn fetch_with_retry(
    client: &Client,
    url: &str,
    max_retries: u32,
) -> Result<String, Box<dyn std::error::Error>> {
    let mut last_error = None;

    for attempt in 0..max_retries {
        match client.get(url).send() {
            Ok(resp) => {
                let status = resp.status();
                if status.is_success() {
                    return Ok(resp.text()?);
                }
                if status.as_u16() == 429 || status.is_server_error() {
                    let wait = Duration::from_secs(2u64.pow(attempt));
                    eprintln!("[{}] ステータス{}、{}秒後にリトライ", url, status, wait.as_secs());
                    thread::sleep(wait);
                    last_error = Some(format!("HTTP {}", status));
                    continue;
                }
                return Err(format!("HTTP {}", status).into());
            }
            Err(e) => {
                let wait = Duration::from_secs(2u64.pow(attempt));
                eprintln!("[{}] エラー: {}、{}秒後にリトライ", url, e, wait.as_secs());
                thread::sleep(wait);
                last_error = Some(e.to_string());
            }
        }
    }

    Err(format!("{}回リトライしても失敗: {:?}", max_retries, last_error).into())
}

[dependencies]
csv = "1"
serde = { version = "1", features = ["derive"] }

use csv::Writer;
use serde::Serialize;

#[derive(Serialize)]
struct Article {
    title: String,
    url: String,
}

fn save_to_csv(articles: &[Article], path: &str) -> Result<(), Box<dyn std::error::Error>> {
    let mut writer = Writer::from_path(path)?;
    for article in articles {
        writer.serialize(article)?;
    }
    writer.flush()?;
    Ok(())
}

use serde::Serialize;
use std::fs;

#[derive(Serialize)]
struct Article {
    title: String,
    url: String,
}

fn save_to_json(articles: &[Article], path: &str) -> Result<(), Box<dyn std::error::Error>> {
    let json = serde_json::to_string_pretty(articles)?;
    fs::write(path, json)?;
    Ok(())
}

use reqwest::blocking::get;

fn check_robots_txt(domain: &str) -> Result<(), Box<dyn std::error::Error>> {
    let url = format!("{}/robots.txt", domain);
    let body = get(&url)?.text()?;
    println!("=== robots.txt ===\n{}", body);
    Ok(())
}

use reqwest::blocking::Client;
use scraper::{Html, Selector};
use std::time::Duration;
use std::thread;
use serde::Serialize;

#[derive(Serialize, Debug)]
struct Article {
    title: String,
    url: String,
}

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let client = Client::builder()
        .user_agent("Mozilla/5.0 (compatible; RustScraper/1.0)")
        .timeout(Duration::from_secs(30))
        .build()?;

    let base_url = "https://zenn.dev/topics/rust";
    let body = client.get(base_url).send()?.text()?;
    let document = Html::parse_document(&body);

    let article_selector = Selector::parse("article a").unwrap();
    let mut articles: Vec<Article> = Vec::new();

    for element in document.select(&article_selector) {
        if let Some(href) = element.value().attr("href") {
            let title: String = element.text().collect();
            let title = title.trim().to_string();
            if !title.is_empty() && href.starts_with("/") {
                articles.push(Article {
                    title,
                    url: format!("https://zenn.dev{}", href),
                });
            }
        }
    }

    // 重複除去
    articles.dedup_by(|a, b| a.url == b.url);

    for article in &articles {
        println!("{}: {}", article.title, article.url);
    }

    // JSON出力
    let json = serde_json::to_string_pretty(&articles)?;
    std::fs::write("articles.json", json)?;
    println!("\n{}件の記事を保存しました", articles.len());

    Ok(())
}