some library updates and adds support for adblock lists

cmd/bind/get-remote-data.go

@@ -3,15 +3,17 @@ package main
 import (
 	"time"
 
-	"pihole-blocklist/bind/internal/httpclient"
+	"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/httpclient"
+	"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/log"
 )
 
 func getListData() []string {
 	var badDomains []string
 	listSimple := make(chan []string)
 	listComplex := make(chan []string)
+	listAdBlock := make(chan []string)
 
-	cfg.Log.Info("downloading blocklists")
+	log.Info("downloading blocklists")
 	// Get Simple Blocklists
 	go func() {
 		data := getData(cfg.ConfigFile.Sources.DomainListURLs)
@@ -26,28 +28,39 @@ func getListData() []string {
 		listComplex <- domains
 	}()
 
+	// Get AdBlock Blocklists
+	go func() {
+		data := getData(cfg.ConfigFile.Sources.AdBlockURLs)
+		domains := parseAdBlock(data)
+		listAdBlock <- domains
+	}()
+
 	// Wait for all downloads to finish
 	var (
-		simple, complex                 []string
-		simpleFinished, complexFinished bool
+		simple, complex, adblock                         []string
+		simpleFinished, complexFinished, adBlockFinished bool
 	)
 
 	for {
 		select {
 		case simple = <-listSimple:
+			log.Info("all simple lists downloaded")
 			simpleFinished = true
-			cfg.Log.Info("all simple lists downloaded")
 		case complex = <-listComplex:
-			cfg.Log.Info("all complex lists downloaded")
+			log.Info("all complex lists downloaded")
 			complexFinished = true
+		case adblock = <-listAdBlock:
+			log.Info("all adblock lists downloaded")
+			adBlockFinished = true
 		default:
 			time.Sleep(time.Millisecond * 100)
 		}
 
-		if simpleFinished && complexFinished {
+		if simpleFinished && complexFinished && adBlockFinished {
 			badDomains = append(badDomains, simple...)
 			badDomains = append(badDomains, complex...)
-			cfg.Log.Info("domains retrieved", "hosts", len(badDomains))
+			badDomains = append(badDomains, adblock...)
+			log.Info("domains retrieved", "hosts", len(badDomains))
 			break
 		}
 	}
@@ -62,11 +75,11 @@ func getData(urls []string) []byte {
 	listData := make([]byte, 0, len(urls)+1)
 
 	for _, u := range urls {
-		cfg.Log.Debug("downloading", "url", u)
+		log.Debug("downloading", "url", u)
 		c := httpclient.DefaultClient()
 		data, err := c.Get(u)
 		if err != nil {
-			cfg.Log.Error("unable to get remote content", "error", err, "url", err)
+			log.Error("unable to get remote content", "error", err, "url", err)
 		}
 		listData = append(listData, data...)
 		// add newline to the end of data, you know, for funzies