依赖
com.squareup.okhttp3
okhttp
3.14.9
com.squareup.okio
okio
2.6.0
org.jsoup
jsoup
1.13.1
commons-io
commons-io
2.8.0
工具封装
import okhttp3.Call;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import org.apache.commons.io.FileUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import java.io.File;
import java.io.InputStream;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.util.concurrent.TimeUnit;
public class HttpUtil {
private static final String HOST = "127.0.0.1";
private static final int PORT = 1080;
private HttpUtil() {}
static {
System.setProperty("https.protocols","TLSv1,TLSv1.1,TLSv1.2");
}
/**
* 创建okHttp客户端
*/
private static OkHttpClient okHttpClient = new OkHttpClient.Builder()
.connectTimeout(60, TimeUnit.SECONDS)
.readTimeout(60,TimeUnit.SECONDS)
.followRedirects(true)
//有代理就使用代理,没有代理注释掉
.proxy(new Proxy(Proxy.Type.HTTP, new InetSocketAddress(HOST, PORT)))
.build();
/**
* 结合jsoup爬取dom,jsoup内部使用httpclient,效率不好
* @param url
* @param host
* @return
* @throws Exception
*/
public static Document getDoc(String url,String host) throws Exception {
Request request = new Request.Builder().url(url).get().build();
Call call = okHttpClient.newCall(request);
String body = call.execute().body().string();
return Jsoup.parse(body,host);
}
/**
* 下载网络文件
* @param url
* @param file
* @throws Exception
*/
public static void download(String url,File file) throws Exception {
Request request = new Request.Builder().url(url).get().build();
Call call = okHttpClient.newCall(request);
try(InputStream inputStream = call.execute().body().byteStream()) {
FileUtils.copyInputStreamToFile(inputStream,file);
}catch (Exception e){
throw e;
}
}
}