multiple updates
This commit is contained in:
@ -1,29 +1,27 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"pihole-blocklist/v2/internal/httpclient"
|
||||
"time"
|
||||
|
||||
"pihole-blocklist/bind/internal/httpclient"
|
||||
)
|
||||
|
||||
func getListData() []string {
|
||||
defer timeTrack(time.Now(), "getListData")
|
||||
|
||||
var badDomains []string
|
||||
listSimple := make(chan []string)
|
||||
listComplex := make(chan []string)
|
||||
|
||||
log.Printf("[INFO] Downloading blocklists\n")
|
||||
cfg.Log.Info("downloading blocklists")
|
||||
// Get Simple Blocklists
|
||||
go func() {
|
||||
data := getData(config.Config.Sources.DomainListURLs)
|
||||
data := getData(cfg.ConfigFile.Sources.DomainListURLs)
|
||||
domains := parseSimple(data)
|
||||
listSimple <- domains
|
||||
}()
|
||||
|
||||
// Get Host File Blocklists
|
||||
go func() {
|
||||
data := getData(config.Config.Sources.HostFileURLs)
|
||||
data := getData(cfg.ConfigFile.Sources.HostFileURLs)
|
||||
domains := parseComplex(data)
|
||||
listComplex <- domains
|
||||
}()
|
||||
@ -38,9 +36,9 @@ func getListData() []string {
|
||||
select {
|
||||
case simple = <-listSimple:
|
||||
simpleFinished = true
|
||||
log.Printf("[INFO] All simple lists have been retrieved.\n")
|
||||
cfg.Log.Info("all simple lists downloaded")
|
||||
case complex = <-listComplex:
|
||||
log.Printf("[INFO] All complex lists have been retrieved.\n")
|
||||
cfg.Log.Info("all complex lists downloaded")
|
||||
complexFinished = true
|
||||
default:
|
||||
time.Sleep(time.Millisecond * 100)
|
||||
@ -49,28 +47,26 @@ func getListData() []string {
|
||||
if simpleFinished && complexFinished {
|
||||
badDomains = append(badDomains, simple...)
|
||||
badDomains = append(badDomains, complex...)
|
||||
log.Printf("[INFO] Number of domains detected: %d\n", len(badDomains))
|
||||
cfg.Log.Info("domains retrieved", "hosts", len(badDomains))
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// append deny list items to list of blocked domains
|
||||
badDomains = append(badDomains, config.Config.DenyList...)
|
||||
badDomains = append(badDomains, cfg.ConfigFile.DenyList...)
|
||||
|
||||
return badDomains
|
||||
}
|
||||
|
||||
func getData(urls []string) []byte {
|
||||
defer timeTrack(time.Now(), "getData")
|
||||
|
||||
var listData []byte
|
||||
listData := make([]byte, 0, len(urls)+1)
|
||||
|
||||
for _, u := range urls {
|
||||
log.Printf("[TRACE] Downloading URL: %s\n", u)
|
||||
cfg.Log.Debug("downloading", "url", u)
|
||||
c := httpclient.DefaultClient()
|
||||
data, err := c.Get(u)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Unable to get remote content from URL (%s): %v", u, err)
|
||||
cfg.Log.Error("unable to get remote content", "error", err, "url", err)
|
||||
}
|
||||
listData = append(listData, data...)
|
||||
// add newline to the end of data, you know, for funzies
|
||||
|
Reference in New Issue
Block a user