multiple updates

This commit is contained in:
2023-12-09 14:21:28 -06:00
parent a11f92745d
commit 881c11b910
23 changed files with 721 additions and 433 deletions

View File

@ -1,29 +1,27 @@
package main
import (
"log"
"pihole-blocklist/v2/internal/httpclient"
"time"
"pihole-blocklist/bind/internal/httpclient"
)
func getListData() []string {
defer timeTrack(time.Now(), "getListData")
var badDomains []string
listSimple := make(chan []string)
listComplex := make(chan []string)
log.Printf("[INFO] Downloading blocklists\n")
cfg.Log.Info("downloading blocklists")
// Get Simple Blocklists
go func() {
data := getData(config.Config.Sources.DomainListURLs)
data := getData(cfg.ConfigFile.Sources.DomainListURLs)
domains := parseSimple(data)
listSimple <- domains
}()
// Get Host File Blocklists
go func() {
data := getData(config.Config.Sources.HostFileURLs)
data := getData(cfg.ConfigFile.Sources.HostFileURLs)
domains := parseComplex(data)
listComplex <- domains
}()
@ -38,9 +36,9 @@ func getListData() []string {
select {
case simple = <-listSimple:
simpleFinished = true
log.Printf("[INFO] All simple lists have been retrieved.\n")
cfg.Log.Info("all simple lists downloaded")
case complex = <-listComplex:
log.Printf("[INFO] All complex lists have been retrieved.\n")
cfg.Log.Info("all complex lists downloaded")
complexFinished = true
default:
time.Sleep(time.Millisecond * 100)
@ -49,28 +47,26 @@ func getListData() []string {
if simpleFinished && complexFinished {
badDomains = append(badDomains, simple...)
badDomains = append(badDomains, complex...)
log.Printf("[INFO] Number of domains detected: %d\n", len(badDomains))
cfg.Log.Info("domains retrieved", "hosts", len(badDomains))
break
}
}
// append deny list items to list of blocked domains
badDomains = append(badDomains, config.Config.DenyList...)
badDomains = append(badDomains, cfg.ConfigFile.DenyList...)
return badDomains
}
func getData(urls []string) []byte {
defer timeTrack(time.Now(), "getData")
var listData []byte
listData := make([]byte, 0, len(urls)+1)
for _, u := range urls {
log.Printf("[TRACE] Downloading URL: %s\n", u)
cfg.Log.Debug("downloading", "url", u)
c := httpclient.DefaultClient()
data, err := c.Get(u)
if err != nil {
log.Printf("[ERROR] Unable to get remote content from URL (%s): %v", u, err)
cfg.Log.Error("unable to get remote content", "error", err, "url", err)
}
listData = append(listData, data...)
// add newline to the end of data, you know, for funzies