91 lines
2.2 KiB
Go
91 lines
2.2 KiB
Go
package main
|
|
|
|
import (
|
|
"time"
|
|
|
|
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/httpclient"
|
|
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/log"
|
|
)
|
|
|
|
func getListData() []string {
|
|
var badDomains []string
|
|
listSimple := make(chan []string)
|
|
listComplex := make(chan []string)
|
|
listAdBlock := make(chan []string)
|
|
|
|
log.Info("downloading blocklists")
|
|
// Get Simple Blocklists
|
|
go func() {
|
|
data := getData(cfg.ConfigFile.Sources.DomainListURLs)
|
|
domains := parseSimple(data)
|
|
listSimple <- domains
|
|
}()
|
|
|
|
// Get Host File Blocklists
|
|
go func() {
|
|
data := getData(cfg.ConfigFile.Sources.HostFileURLs)
|
|
domains := parseComplex(data)
|
|
listComplex <- domains
|
|
}()
|
|
|
|
// Get AdBlock Blocklists
|
|
go func() {
|
|
data := getData(cfg.ConfigFile.Sources.AdBlockURLs)
|
|
domains := parseAdBlock(data)
|
|
listAdBlock <- domains
|
|
}()
|
|
|
|
// Wait for all downloads to finish
|
|
var (
|
|
simple, complex, adblock []string
|
|
simpleFinished, complexFinished, adBlockFinished bool
|
|
)
|
|
|
|
for {
|
|
select {
|
|
case simple = <-listSimple:
|
|
log.Info("all simple lists downloaded")
|
|
simpleFinished = true
|
|
case complex = <-listComplex:
|
|
log.Info("all complex lists downloaded")
|
|
complexFinished = true
|
|
case adblock = <-listAdBlock:
|
|
log.Info("all adblock lists downloaded")
|
|
adBlockFinished = true
|
|
default:
|
|
time.Sleep(time.Millisecond * 100)
|
|
}
|
|
|
|
if simpleFinished && complexFinished && adBlockFinished {
|
|
badDomains = append(badDomains, simple...)
|
|
badDomains = append(badDomains, complex...)
|
|
badDomains = append(badDomains, adblock...)
|
|
log.Info("domains retrieved", "hosts", len(badDomains))
|
|
break
|
|
}
|
|
}
|
|
|
|
// append deny list items to list of blocked domains
|
|
badDomains = append(badDomains, cfg.ConfigFile.DenyList...)
|
|
|
|
return badDomains
|
|
}
|
|
|
|
func getData(urls []string) []byte {
|
|
listData := make([]byte, 0, len(urls)+1)
|
|
|
|
for _, u := range urls {
|
|
log.Debug("downloading", "url", u)
|
|
c := httpclient.DefaultClient()
|
|
data, err := c.Get(u)
|
|
if err != nil {
|
|
log.Error("unable to get remote content", "error", err, "url", err)
|
|
}
|
|
listData = append(listData, data...)
|
|
// add newline to the end of data, you know, for funzies
|
|
listData = append(listData, '\n')
|
|
}
|
|
|
|
return listData
|
|
}
|