异步、多线程、Java爬取某网站图片

一、网页图片爬取类

package com.yhyl.utils;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Component;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 * @program: springboot-sqlserver-elasticsearch-api
 * @description
 * @author: xbwen
 * @create: 2021-07-22 16:40
 **/
@Component
public class HtmlParseUtil {
    public static void main(String[] args) throws IOException {
        String url = "https://www.yeitu.com/meinv/xinggan/20180919_14722.html";
        Set urlList = new HashSet<>();
        urlList.add(url);
        ExecutorService executorService = Executors.newFixedThreadPool(24);
        Set parseUrlList = new HtmlParseUtil().parseUrl(urlList);
        Set dataList = new HtmlParseUtil().parseData(parseUrlList,executorService);
        executorService.shutdown();
    }

    //解析网页地址，递归模拟浏览器网页请求，获取自己想要得页面数
    public Set parseUrl(Set urlList) throws  IOException{
        if(urlList.size()>=50){
            return urlList;
        }
        Set tempList = new HashSet<>();
        for (String url : urlList) {
            Document document = Jsoup.parse(new URL(url), 30000);
            Elements elements = document.getElementsByTag("a");
            for (Element element : elements) {
                String href = element.attr("href");
                if(href.endsWith(".html")){
                    tempList.add(href);
                }
            }
        }
        urlList.addAll(tempList);
        urlList = parseUrl(urlList);
        return urlList;
    }

    //根据网页地址，模拟浏览器请求，获取页面图片链接地址，并下载到本地磁盘
    public Set parseData(Set urlList, ExecutorService executorService){
        Set contents = new HashSet<>();
        for (String url : urlList) {
            executorService.execute(new Thread() {
                @Override
                public synchronized void run(){
                    try {
                        Document document = Jsoup.parse(new URL(url), 30000);
                        Elements imgBoxElements = document.getElementsByClass("img_box");
                        for (Element element : imgBoxElements) {
                            Elements imgElements = element.getElementsByTag("img");
                            for (Element imgElement : imgElements) {
                                String title = imgElement.attr("alt");
                                String img = imgElement.attr("src");
                                System.out.println("下载完成:"+title+"@"+img);
//                                HtmlParseUtil htmlParseUtilProxy = SpringContextHolder.getBean(HtmlParseUtil.class);
                                downloadPicture(title,img,"E:\\picture\\"+title+".jpg");
//                                contents.add(title+"@"+img);
                            }
                        }
                    }catch (Exception e){
                        e.printStackTrace();
                    }

                }
            });
        }
        return contents;
    }

    //链接url下载图片
    @Async
    public void downloadPicture(String imgTitle, String imgUrl, String path) throws Exception{
        URL url = new URL(imgUrl);
        // 打开连接
        URLConnection conn = url.openConnection();
//            HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        conn.setConnectTimeout(50000);
        conn.setReadTimeout(50000);
//            conn.setRequestProperty(":authority", "file.jiutuvip.com");
//            conn.setRequestProperty(":method", "GET");
//            conn.setRequestProperty(":path", "/2021/0105/20210105101307685.jpg");
//            conn.setRequestProperty(":scheme","https");
        conn.setRequestProperty("accept","image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8");
        conn.setRequestProperty("accept-encoding", "gzip, deflate, br");
        conn.setRequestProperty("accept-language", "zh-CN,zh;q=0.9");
        conn.setRequestProperty("if-modified-since", "Tue, 05 Jan 2021 02:13:07 GMT");
        conn.setRequestProperty("if-none-match", "5ff3cb33-6aa5");
        conn.setRequestProperty("referer", "https://www.yeitu.net/");
        conn.setRequestProperty("sec-ch-ua", " Not;A Brand;"+"v=\"99\", \"Google Chrome;"+"v=\"91\", \"Chromium;"+"v=\"91\"");
        conn.setRequestProperty("sec-ch-ua-mobile", "?0");
        conn.setRequestProperty("sec-fetch-dest", "image");
        conn.setRequestProperty("sec-fetch-mode", "no-cors");
        conn.setRequestProperty("sec-fetch-site", "cross-site");
        conn.setRequestProperty("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36");
        conn.connect();

//        List