package main import ( "time" "gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/httpclient" "gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/log" ) func getListData() []string { var badDomains []string listSimple := make(chan []string) listComplex := make(chan []string) listAdBlock := make(chan []string) log.Info("downloading blocklists") // Get Simple Blocklists go func() { data := getData(cfg.ConfigFile.Sources.DomainListURLs) domains := parseSimple(data) listSimple <- domains }() // Get Host File Blocklists go func() { data := getData(cfg.ConfigFile.Sources.HostFileURLs) domains := parseComplex(data) listComplex <- domains }() // Get AdBlock Blocklists go func() { data := getData(cfg.ConfigFile.Sources.AdBlockURLs) domains := parseAdBlock(data) listAdBlock <- domains }() // Wait for all downloads to finish var ( simple, complex, adblock []string simpleFinished, complexFinished, adBlockFinished bool ) for { select { case simple = <-listSimple: log.Info("all simple lists downloaded") simpleFinished = true case complex = <-listComplex: log.Info("all complex lists downloaded") complexFinished = true case adblock = <-listAdBlock: log.Info("all adblock lists downloaded") adBlockFinished = true default: time.Sleep(time.Millisecond * 100) } if simpleFinished && complexFinished && adBlockFinished { badDomains = append(badDomains, simple...) badDomains = append(badDomains, complex...) badDomains = append(badDomains, adblock...) log.Info("domains retrieved", "hosts", len(badDomains)) break } } // append deny list items to list of blocked domains badDomains = append(badDomains, cfg.ConfigFile.DenyList...) return badDomains } func getData(urls []string) []byte { listData := make([]byte, 0, len(urls)+1) for _, u := range urls { log.Debug("downloading", "url", u) c := httpclient.DefaultClient() data, err := c.Get(u) if err != nil { log.Error("unable to get remote content", "error", err, "url", err) } listData = append(listData, data...) // add newline to the end of data, you know, for funzies listData = append(listData, '\n') } return listData }