package main import ( "log" "pihole-blocklist/v2/internal/httpclient" "time" ) func getListData() []string { defer timeTrack(time.Now(), "getListData") var badDomains []string listSimple := make(chan []string) listComplex := make(chan []string) log.Printf("[INFO] Downloading blocklists\n") // Get Simple Blocklists go func() { data := getData(config.Config.Sources.DomainListURLs) domains := parseSimple(data) listSimple <- domains }() // Get Host File Blocklists go func() { data := getData(config.Config.Sources.HostFileURLs) domains := parseComplex(data) listComplex <- domains }() // Wait for all downloads to finish var ( simple, complex []string simpleFinished, complexFinished bool ) for { select { case simple = <-listSimple: simpleFinished = true log.Printf("[INFO] All simple lists have been retrieved.\n") case complex = <-listComplex: log.Printf("[INFO] All complex lists have been retrieved.\n") complexFinished = true default: time.Sleep(time.Millisecond * 100) } if simpleFinished && complexFinished { badDomains = append(badDomains, simple...) badDomains = append(badDomains, complex...) log.Printf("[INFO] Number of domains detected: %d\n", len(badDomains)) break } } // append deny list items to list of blocked domains badDomains = append(badDomains, config.Config.DenyList...) return badDomains } func getData(urls []string) []byte { defer timeTrack(time.Now(), "getData") var listData []byte for _, u := range urls { log.Printf("[TRACE] Downloading URL: %s\n", u) c := httpclient.DefaultClient() data, err := c.Get(u) if err != nil { log.Printf("[ERROR] Unable to get remote content from URL (%s): %v", u, err) } listData = append(listData, data...) // add newline to the end of data, you know, for funzies listData = append(listData, '\n') } return listData }