package main import ( "time" "pihole-blocklist/bind/internal/httpclient" ) func getListData() []string { var badDomains []string listSimple := make(chan []string) listComplex := make(chan []string) cfg.Log.Info("downloading blocklists") // Get Simple Blocklists go func() { data := getData(cfg.ConfigFile.Sources.DomainListURLs) domains := parseSimple(data) listSimple <- domains }() // Get Host File Blocklists go func() { data := getData(cfg.ConfigFile.Sources.HostFileURLs) domains := parseComplex(data) listComplex <- domains }() // Wait for all downloads to finish var ( simple, complex []string simpleFinished, complexFinished bool ) for { select { case simple = <-listSimple: simpleFinished = true cfg.Log.Info("all simple lists downloaded") case complex = <-listComplex: cfg.Log.Info("all complex lists downloaded") complexFinished = true default: time.Sleep(time.Millisecond * 100) } if simpleFinished && complexFinished { badDomains = append(badDomains, simple...) badDomains = append(badDomains, complex...) cfg.Log.Info("domains retrieved", "hosts", len(badDomains)) break } } // append deny list items to list of blocked domains badDomains = append(badDomains, cfg.ConfigFile.DenyList...) return badDomains } func getData(urls []string) []byte { listData := make([]byte, 0, len(urls)+1) for _, u := range urls { cfg.Log.Debug("downloading", "url", u) c := httpclient.DefaultClient() data, err := c.Get(u) if err != nil { cfg.Log.Error("unable to get remote content", "error", err, "url", err) } listData = append(listData, data...) // add newline to the end of data, you know, for funzies listData = append(listData, '\n') } return listData }