2025年Java网络爬取教程（分步指南）

C:\Program Files\Java\jdk-21\bin

java -version

javac -version

├── pom.xml
└── src/
    └── main/
        └── java/
            └── com/
                └── orina/
                    └── scraper/
                        ├── Main.java
                        └── HtmlParser.java
    └── test/
        └── java/
            └── com/
                └── orina/
                    └── scraper/
                        └── MainTest.java

<project>
  <modelVersion>4.0.0</modelVersion>
  <groupId>com.orina</groupId>
  <artifactId>web-scraper</artifactId>
  <version>1.0</version>
  <dependencies>
    <dependency>
      <groupId>org.jsoup</groupId>
      <artifactId>jsoup</artifactId>
      <version>1.15.3</version>
    </dependency>
  </dependencies>
</project>

plugins {
    id 'java'
}

group = 'com.orina'
version = '1.0'

repositories {
    mavenCentral()
}

dependencies {
    implementation 'org.jsoup:jsoup:1.15.3'
}

C:\Program Files\Gradle

%GRADLE_HOME%\bin

gradle -v

plugins {
    id 'java'
}

group = 'com.tutorial'
version = '1.0'

repositories {
    mavenCentral()
}

dependencies {
    implementation 'org.jsoup:jsoup:1.15.3'
}

implementation 'org.seleniumhq.selenium:selenium-java:4.21.0'
implementation 'io.github.bonigarcia:webdrivermanager:5.8.0'

plugins {
    id 'java'
}

group = 'com.tutorial'
version = '1.0'

repositories {
    mavenCentral()
}

dependencies {
    // Scraping libraries
    implementation 'org.jsoup:jsoup:1.17.2'
    implementation 'org.apache.httpcomponents:httpclient:4.5.14'
    implementation 'com.fasterxml.jackson.core:jackson-databind:2.17.2'
    // Testing
    testImplementation platform('org.junit:junit-bom:5.10.0')
    testImplementation 'org.junit.jupiter:junit-jupiter'
}
test {
    useJUnitPlatform()
}

{
  "startUrl": "https://news.ycombinator.com/",
  "pages": 3,
  "minDelayMs": 1200,
  "maxDelayMs": 2500,
  "retries": 3,
  "output": "hn_results.csv",
  "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 OrinaScraper/1.0"
}

proxies.txt

ultra.marsproxies.com:44443:mr45184psO8:MKRYlpZgSa_country-us_city-losangeles_session-j1ko4q46_lifetime-30m
ultra.marsproxies.com:44443:mr45184psO8:MKRYlpZgSa_country-us_state-alaska_session-v750sry6_lifetime-30m
ultra.marsproxies.com:44443:mr45184psO8:MKRYlpZgSa_country-us_state-kansas_session-afxhhuv5_lifetime-30m
ultra.marsproxies.com:44443:mr45184psO8:MKRYlpZgSa_country-us_state-maryland_session-u8sgsjv6_lifetime-30m
ultra.marsproxies.com:44443:mr45184psO8:MKRYlpZgSa_country-us_city-oakland_session-qlzwp7xo_lifetime-30m

com.tutorial

Ycombinatorblogpostextractor

package com.tutorial;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.DefaultProxyRoutePlanner;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;

import java.io.*;
import java.nio.charset.StandardCharsets;
import java.security.SecureRandom;
import java.util.*;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;

public class HackerNewsScraper {
    // Config
    static class Config {
        String startUrl;
        int pages;
        int minDelayMs;
        int maxDelayMs;
        int retries;
        String output;
        String userAgent;
    }

    // Proxy entry
    static class ProxyEntry {
        final String host;
        final int port;
        final String username;
        final String password; // may include session + geo tokens (keep as-is)
        final String label;

        ProxyEntry(String host, int port, String username, String password, String label) {
            this.host = host; this.port = port; this.username = username; this.password = password; this.label = label;
        }

        static ProxyEntry parse(String line) {
            // Expect format host:port:user:passWithSessionAndFlags
            String[] parts = line.trim().split(":", 4);
            if (parts.length < 4) throw new IllegalArgumentException("Bad proxy line: " + line);
            String host = parts[0];
            int port = Integer.parseInt(parts[1]);
            String user = parts[2];
            String pass = parts[3];
            // derive a short label from any _city- or _state- token if present
            String label = "proxy";
            int cityIdx = pass.indexOf("_city-");
            int stateIdx = pass.indexOf("_state-");
            if (cityIdx >= 0) {
                String rest = pass.substring(cityIdx + 6);
                label = rest.split("[_\\s]")[0];
            } else if (stateIdx >= 0) {
                String rest = pass.substring(stateIdx + 7);
                label = rest.split("[_\\s]")[0];
            }

            return new ProxyEntry(host, port, user, pass, label);
        }
    }

    private final Config cfg;
    private final List<ProxyEntry> proxies;
    private final AtomicInteger rr = new AtomicInteger(0);
    private final SecureRandom rng = new SecureRandom();

    public HackerNewsScraper(Config cfg, List<ProxyEntry> proxies) {
        this.cfg = cfg; this.proxies = proxies;
        if (proxies.isEmpty()) throw new IllegalStateException("No proxies loaded");
    }

    public static void main(String[] args) throws Exception {
        Config cfg = loadConfig();
        List<ProxyEntry> proxies = loadProxies();
        HackerNewsScraper app = new HackerNewsScraper(cfg, proxies);
        app.scrape();
    }

    private void scrape() throws Exception {
        try (PrintWriter out = new PrintWriter(new OutputStreamWriter(new FileOutputStream(cfg.output), StandardCharsets.UTF_8))) 
            out.println("postId,rank,title,link,timeText,timeISO,commentsCount,commentsUrl");

            String url = cfg.startUrl;
            for (int p = 1; p <= cfg.pages; p++) {
                Document doc = getWithRetries(url);
                if (doc == null) throw new IOException("Failed to fetch: " + url);

                // Parse items on the page
                for (Element row : doc.select("tr.athing")) {
                    String postId = row.attr("data-id").trim();

                    // HN uses <span class="titleline"><a ...> now; keep fallback for .storylink
                    Element titleA = Optional.ofNullable(row.selectFirst("span.titleline > a"))
                            .orElse(row.selectFirst(".storylink"));
                    if (titleA == null) continue;
                    String title = titleA.text();
                    String link = titleA.attr("href");

                    Element metaRow = row.nextElementSibling();
                    String timeText = "";
                    String timeISO = "";
                    int commentsCount = 0;
                    String commentsUrl = "";

                    if (metaRow != null) {
                        Element sub = metaRow.selectFirst(".subtext");
                        if (sub != null) {
                            Element age = sub.selectFirst(".age a");
                            if (age != null) {
                                timeText = age.text();
                                // 'title' attribute contains absolute time (e.g., 2025-09-29T08:12:34)
                                timeISO = age.attr("title");
                            }
                            List<Element> itemLinks = sub.select("a[href^=item?id]");
                            if (!itemLinks.isEmpty()) {
                                Element c = itemLinks.get(itemLinks.size() - 1);
                                commentsUrl = abs("https://news.ycombinator.com/", c.attr("href"));
                                String txt = c.text();
                                if (txt.matches(".*\\d+.*")) {
                                    String digits = txt.replaceAll("\\D+", "");
                                    if (!digits.isEmpty()) commentsCount = Integer.parseInt(digits);
                                } else {
                                    commentsCount = 0; // "discuss"
                                }
                            }
                        }
                    }

                    String rank = row.selectFirst("span.rank") != null ? row.selectFirst("span.rank").text() : "";
                    out.printf(Locale.ROOT,
                            "%s,%s,\"%s\",%s,%s,%s,%d,%s%n",
                            csv(postId), csv(rank), csv(title), csv(link), csv(timeText), csv(timeISO), commentsCount, csv(commentsUrl));
                }

                // Find pagination (More)
                Element more = doc.selectFirst("a.morelink");
                if (p < cfg.pages && more != null) {
                    url = abs(cfg.startUrl, more.attr("href"));
                    sleepJitter(cfg.minDelayMs, cfg.maxDelayMs);
                } else {
                    break;
                }
            }
        }
        System.out.println("Done. Wrote " + cfg.output);
    }

    // --- Networking with rotating proxies ---
    private Document getWithRetries(String url) {
        int attempts = 0;
        while (attempts < cfg.retries) {
            ProxyEntry px = pickProxy();
            try {
                Document d = fetchViaProxy(px, url);
                if (d != null) return d;
            } catch (Exception e) {
                // log and fall through to retry with next proxy
                System.err.println("[WARN] " + px.label + " failed: " + e.getMessage());
            }
            attempts++;
            sleep(cfg.minDelayMs + 400L);
        }
        return null;
    }

    private ProxyEntry pickProxy() {
        int i = Math.floorMod(rr.getAndIncrement(), proxies.size());
        return proxies.get(i);
    }

    private Document fetchViaProxy(ProxyEntry px, String url) throws Exception {
        CredentialsProvider credsProvider = new BasicCredentialsProvider();
        credsProvider.setCredentials(new AuthScope(px.host, px.port), new UsernamePasswordCredentials(px.username, px.password));

        HttpHost proxy = new HttpHost(px.host, px.port);
        DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);

        try (CloseableHttpClient client = HttpClients.custom()
                .setDefaultCredentialsProvider(credsProvider)
                .setRoutePlanner(routePlanner)
                .build()) {

            HttpGet get = new HttpGet(url);
            get.setHeader("User-Agent", cfg.userAgent);
            get.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
            get.setHeader("Accept-Encoding", "gzip, deflate");
            get.setHeader("Accept-Language", "en-US,en;q=0.9");

            try (CloseableHttpResponse resp = client.execute(get)) {
                int status = resp.getStatusLine().getStatusCode();
                if (status >= 200 && status < 300) {
                    String html = EntityUtils.toString(resp.getEntity(), StandardCharsets.UTF_8);
                    return Jsoup.parse(html, url);
                }
                throw new IOException("HTTP " + status);
            }
        }
    }

    // --- Utilities ---
    private static String abs(String base, String href) {
        try { return new java.net.URL(new java.net.URL(base), href).toString(); }
        catch (Exception e) { return href; }
    }

    private static String csv(String s) {
        if (s == null) return "";
        String t = s.replace("\"", "\"\"");
        if (t.contains(",") || t.contains("\n") || t.contains("\"")) return '"' + t + '"';
        return t;
    }

    private void sleepJitter(int minMs, int maxMs) {
        int delta = Math.max(0, maxMs - minMs);
        int wait = minMs + (delta == 0 ? 0 : ThreadLocalRandom.current().nextInt(delta));
        sleep(wait);
    }

    private void sleep(long ms) {
        try { Thread.sleep(ms); } catch (InterruptedException ignored) { }
    }

    private static Config loadConfig() throws IOException {
        ObjectMapper om = new ObjectMapper();
        try (InputStream in = HackerNewsScraper.class.getClassLoader().getResourceAsStream("config.json")) {
            if (in == null) throw new FileNotFoundException("config.json not found in resources");
            JsonNode node = om.readTree(in);
            Config c = new Config();
            c.startUrl = node.get("startUrl").asText();
            c.pages = node.get("pages").asInt(1);
            c.minDelayMs = node.get("minDelayMs").asInt(1200);
            c.maxDelayMs = node.get("maxDelayMs").asInt(2500);
            c.retries = node.get("retries").asInt(3);
            c.output = node.get("output").asText("hn_results.csv");
            c.userAgent = node.get("userAgent").asText("Mozilla/5.0 OrinaScraper");
            return c;
        }
    }

    private static List<ProxyEntry> loadProxies() throws IOException {
        InputStream in = HackerNewsScraper.class.getClassLoader().getResourceAsStream("proxies.txt");
        if (in == null) throw new FileNotFoundException("proxies.txt not found in resources");
        List<String> lines;
        try (BufferedReader br = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) {
            lines = br.lines().map(String::trim).filter(s -> !s.isEmpty() && !s.startsWith("#")).collect(Collectors.toList());
        }
        List<ProxyEntry> list = new ArrayList<>();
        for (String line : lines) list.add(ProxyEntry.parse(line));
        return list;
    }
}

package com.tutorial;