go语言抓取文件中整个URL（可递归目录）

可递归

package Createfile

import (
    "fmt"
    "log"
    "os"
    "os/exec"
    "sync"
    "time"
)

var create sync.Once

func createdir()  {
    cmd := exec.Command("mkdir","OutPut")
    if _,cmderr := cmd.CombinedOutput();cmderr !=nil{
        log.Println(cmderr)
    }
}

func CreateFile() *os.File {
    create.Do(func() {
        f,err := os.Stat("OutPut")
        if !os.IsExist(err){
            createdir()
        }
        if f.IsDir(){
            log.Println("OutPut目录已经存在")
        }
        createdir()
    })

    osfile,err := os.OpenFile("OutPut/geturl" + fmt.Sprintf("-%d%d%d%d%d%d.log",
        time.Now().Year(),
        time.Now().Month(),
        time.Now().Day(),
        time.Now().Hour(),
        time.Now().Minute(),
        time.Now().Second(),
    ),os.O_CREATE|os.O_WRONLY|os.O_APPEND,0644)
    if err != nil{
        log.Println(err)
    }
    return osfile
}

package Readfile

import (
    "io/ioutil"
    "log"
)

func ReadFile(filename string) []byte {
    filebyte,fileerr := ioutil.ReadFile(filename)
    if fileerr != nil {
        log.Println(fileerr)
    }
    return filebyte
}

package Treelist

import (
    "fmt"
    "io/ioutil"
    "path"
)

var FileSlice []string

func Recursion(dir string) {

    files, errRead := ioutil.ReadDir(dir)
    if errRead != nil {
        fmt.Println(errRead)
    }
    for _, file := range files {
        if file.IsDir() {
            filename := file.Name()
            path := dir + filename + "/"
            Recursion(path)
        }else {
            filename := path.Base(file.Name())
            pwd := dir + filename
            FileSlice = append(FileSlice, pwd)
        }
    }
}

package UrlGet

import (
    "fmt"
    "littletools/GetUrl/Createfile"
    . "littletools/GetUrl/Readfile"
    "log"
    "mvdan.cc/xurls/v2"
    "regexp"
)

func UrlGet(filename string) {
    file := Createfile.CreateFile()
    defer file.Close()
    arraybyte := xurls.Relaxed().FindAll(ReadFile(filename), -1)
    for k, v := range arraybyte {
        if match,err := regexp.MatchString(`^(https://){1}|^(http://){1}`,string(v));match{
            if err!=nil{
                log.Println(err)
            }
            if matchgov,errgov := regexp.MatchString(`(匹配不要的正则){1}`,string(v));matchgov{
                if errgov != nil{
                    log.Println(errgov)
                }
                continue
            }
            file.WriteString(fmt.Sprintf(" %s %d %s\n",filename, k, v))
        }
    }
}

package main

import (
    "fmt"
    "littletools/GetUrl/Treelist"
    "littletools/GetUrl/UrlGet"
    "os"
    "sync"
)

var wg sync.WaitGroup

//给写入文件加锁，不然会发生并发不安全
var lock sync.Mutex

func main()  {

    if arg_num := len(os.Args[1:]);arg_num < 1{
        fmt.Println("Waring:请输入路径的名称")
        return
    }
    for _, v := range os.Args[1:] {
        dir := v + "/"
        Treelist.Recursion(dir)
    }
    for _, v := range Treelist.FileSlice {
        wg.Add(1)
        go func(filepath string) {
            lock.Lock()
            defer wg.Done()
            defer lock.Unlock()
            UrlGet.UrlGet(filepath)
        }(v)
    }
    wg.Wait()
}

测试目录结构

删除正则二次筛选结果会包含不要的正则部分

加上二次筛选结果不会包含不要的正则部分

纯个人手写思维，转载请注明出处，感谢。