colly 随机UA标识 下载图片

colly  随机UA标识 下载图片

package main

import (
	"io/ioutil"
	"strings"

	"github.com/gocolly/colly"
	"github.com/gocolly/colly/extensions"
)

func dlImg(uri string) {
	logHelper.Info(uri)
	// https://images.hk/upload/goods/20211127152597926.jpg
	c := colly.NewCollector(
	// colly.UserAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3880.400 QQBrowser/10.8.4554.400"),
	// colly.AllowedDomains("foo.com", "bar.com"),
	)
	extensions.RandomUserAgent(c)

	c.OnResponse(func(r *colly.Response) {
		logHelper.Info(r.Request.Headers)
		logHelper.Info(r.StatusCode)
		// logHelper.Info(r.Body)
		bs := r.Body
		fn := func() string {
			l := strings.Split(uri, ".hk/")

			p := "../shop/" + l[1]
			logHelper.Info(l[1], p)
			return p
		}()
		err := ioutil.WriteFile(fn, bs, 0777)
		if err != nil {
			logHelper.Error(err)
		}
	})
	if err := c.Visit(uri); err != nil {
		logHelper.Error(err)
	}

	logHelper.Info("end")
}

func dlImgFromFile(f string) {
	s := rF(f)
	l := strings.Split(s, "\r\n")
	for _, uri := range l {
		// logHelper.Info(uri)
		dlImg(uri)
	}

}