82 lines
1.9 KiB
Go
82 lines
1.9 KiB
Go
package main
|
|
|
|
import (
|
|
"log"
|
|
"pihole-blocklist/v2/internal/httpclient"
|
|
"time"
|
|
)
|
|
|
|
func getListData() []string {
|
|
defer timeTrack(time.Now(), "getListData")
|
|
|
|
var badDomains []string
|
|
listSimple := make(chan []string)
|
|
listComplex := make(chan []string)
|
|
|
|
log.Printf("[INFO] Downloading blocklists\n")
|
|
// Get Simple Blocklists
|
|
go func() {
|
|
data := getData(config.Config.Sources.DomainListURLs)
|
|
domains := parseSimple(data)
|
|
listSimple <- domains
|
|
}()
|
|
|
|
// Get Host File Blocklists
|
|
go func() {
|
|
data := getData(config.Config.Sources.HostFileURLs)
|
|
domains := parseComplex(data)
|
|
listComplex <- domains
|
|
}()
|
|
|
|
// Wait for all downloads to finish
|
|
var (
|
|
simple, complex []string
|
|
simpleFinished, complexFinished bool
|
|
)
|
|
|
|
for {
|
|
select {
|
|
case simple = <-listSimple:
|
|
simpleFinished = true
|
|
log.Printf("[INFO] All simple lists have been retrieved.\n")
|
|
case complex = <-listComplex:
|
|
log.Printf("[INFO] All complex lists have been retrieved.\n")
|
|
complexFinished = true
|
|
default:
|
|
time.Sleep(time.Millisecond * 100)
|
|
}
|
|
|
|
if simpleFinished && complexFinished {
|
|
badDomains = append(badDomains, simple...)
|
|
badDomains = append(badDomains, complex...)
|
|
log.Printf("[INFO] Number of domains detected: %d\n", len(badDomains))
|
|
break
|
|
}
|
|
}
|
|
|
|
// append deny list items to list of blocked domains
|
|
badDomains = append(badDomains, config.Config.DenyList...)
|
|
|
|
return badDomains
|
|
}
|
|
|
|
func getData(urls []string) []byte {
|
|
defer timeTrack(time.Now(), "getData")
|
|
|
|
var listData []byte
|
|
|
|
for _, u := range urls {
|
|
log.Printf("[TRACE] Downloading URL: %s\n", u)
|
|
c := httpclient.DefaultClient()
|
|
data, err := c.Get(u)
|
|
if err != nil {
|
|
log.Printf("[ERROR] Unable to get remote content from URL (%s): %v", u, err)
|
|
}
|
|
listData = append(listData, data...)
|
|
// add newline to the end of data, you know, for funzies
|
|
listData = append(listData, '\n')
|
|
}
|
|
|
|
return listData
|
|
}
|