Adds configuration from config file.

cmd/bind/get-remote-data.go

@@ -0,0 +1,72 @@
+package main
+
+import (
+	"log"
+	"pihole-blocklist/v2/internal/httpclient"
+	"time"
+)
+
+func getListData() []string {
+	var badDomains []string
+	listSimple := make(chan []string)
+	listComplex := make(chan []string)
+
+	log.Printf("[INFO] Downloading blocklists\n")
+	// Get Simple Blocklists
+	go func() {
+		data := getData(config.Config.Sources.DomainListURLs)
+		domains := parseSimple(data)
+		listSimple <- domains
+	}()
+
+	// Get Host File Blocklists
+	go func() {
+		data := getData(config.Config.Sources.HostFileURLs)
+		domains := parseComplex(data)
+		listComplex <- domains
+	}()
+
+	// Wait for all downloads to finish
+	var (
+		simple, complex                 []string
+		simpleFinished, complexFinished bool
+	)
+
+	for {
+		select {
+		case simple = <-listSimple:
+			simpleFinished = true
+			log.Printf("[INFO] All simple lists have been retrieved.\n")
+		case complex = <-listComplex:
+			log.Printf("[INFO] All complex lists have been retrieved.\n")
+			complexFinished = true
+		default:
+			time.Sleep(time.Millisecond * 100)
+		}
+
+		if simpleFinished && complexFinished {
+			badDomains = append(badDomains, simple...)
+			badDomains = append(badDomains, complex...)
+			log.Printf("[INFO] Number of domains detected: %d\n", len(badDomains))
+			break
+		}
+	}
+
+	return badDomains
+}
+
+func getData(urls []string) []byte {
+	var listData []byte
+
+	for _, u := range urls {
+		log.Printf("[TRACE] Downloading URL: %s\n", u)
+		c := httpclient.DefaultClient()
+		data, err := c.Get(u)
+		if err != nil {
+			log.Printf("[ERROR] Unable to get remote content from URL (%s): %v", u, err)
+		}
+		listData = append(listData, data...)
+	}
+
+	return listData
+}