some library updates and adds support for adblock lists
This commit is contained in:
@@ -5,8 +5,10 @@ import (
|
||||
|
||||
"text/template"
|
||||
|
||||
"pihole-blocklist/bind/assets"
|
||||
"pihole-blocklist/bind/internal/config"
|
||||
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/assets"
|
||||
|
||||
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/common"
|
||||
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/log"
|
||||
)
|
||||
|
||||
func buildBindResponsePolicyFile() error {
|
||||
@@ -25,11 +27,11 @@ func buildBindResponsePolicyFile() error {
|
||||
return err
|
||||
}
|
||||
|
||||
bytesWritten, err := config.WriteFile(cfg.BindOutputFileName, output.Bytes())
|
||||
bytesWritten, err := common.WriteFile(cfg.BindOutputFileName, output.Bytes())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cfg.Log.Debug("file created", "file", cfg.BindOutputFileName, "bytes", bytesWritten)
|
||||
log.Debug("file created", "file", cfg.BindOutputFileName, "bytes", bytesWritten)
|
||||
return nil
|
||||
}
|
||||
|
@@ -3,6 +3,8 @@ package main
|
||||
import (
|
||||
"regexp"
|
||||
"sort"
|
||||
|
||||
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/log"
|
||||
)
|
||||
|
||||
func cleanBadDomains(domains []string) []string {
|
||||
@@ -17,7 +19,7 @@ func cleanBadDomains(domains []string) []string {
|
||||
}
|
||||
}
|
||||
domains = list
|
||||
cfg.Log.Info("hosts removed from blocklist", "reason", "duplicate", "hosts", total-len(domains))
|
||||
log.Info("hosts removed from blocklist", "reason", "duplicate", "hosts", total-len(domains))
|
||||
|
||||
// remove hosts that are too long
|
||||
total = len(domains)
|
||||
@@ -29,7 +31,7 @@ func cleanBadDomains(domains []string) []string {
|
||||
list = append(list, blocklistItem)
|
||||
}
|
||||
domains = list
|
||||
cfg.Log.Info("hosts removed from blocklist", "reason", "too many characters", "hosts", total-len(domains))
|
||||
log.Info("hosts removed from blocklist", "reason", "too many characters", "hosts", total-len(domains))
|
||||
|
||||
// remove allow-listed matches
|
||||
total = len(domains)
|
||||
@@ -39,7 +41,7 @@ func cleanBadDomains(domains []string) []string {
|
||||
for _, allowedItem := range cfg.ConfigFile.AllowLists {
|
||||
_, err := regexp.Compile(allowedItem)
|
||||
if err != nil {
|
||||
cfg.Log.Error("unable to parse allow list item", "error", err, "regex", allowedItem)
|
||||
log.Error("unable to parse allow list item", "error", err, "regex", allowedItem)
|
||||
continue
|
||||
}
|
||||
goodAllowedItemList = append(goodAllowedItemList, allowedItem)
|
||||
@@ -50,7 +52,7 @@ func cleanBadDomains(domains []string) []string {
|
||||
addEntry := true
|
||||
for _, allowedItem := range goodAllowedItemList {
|
||||
if regexp.MustCompile(allowedItem).MatchString(v) {
|
||||
cfg.Log.Debug("hosts removed from blocklist", "reason", "allowed host", "match string", allowedItem, "host", v)
|
||||
log.Debug("hosts removed from blocklist", "reason", "allowed host", "match string", allowedItem, "host", v)
|
||||
addEntry = false
|
||||
}
|
||||
}
|
||||
@@ -59,9 +61,9 @@ func cleanBadDomains(domains []string) []string {
|
||||
}
|
||||
}
|
||||
domains = list
|
||||
cfg.Log.Info("hosts removed from blocklist", "hosts", total-len(domains))
|
||||
log.Info("hosts removed from blocklist", "hosts", total-len(domains))
|
||||
|
||||
cfg.Log.Info("total domains in list", "hosts", len(domains))
|
||||
log.Info("total domains in list", "hosts", len(domains))
|
||||
sort.Strings(domains)
|
||||
return domains
|
||||
}
|
||||
|
@@ -3,15 +3,17 @@ package main
|
||||
import (
|
||||
"time"
|
||||
|
||||
"pihole-blocklist/bind/internal/httpclient"
|
||||
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/httpclient"
|
||||
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/log"
|
||||
)
|
||||
|
||||
func getListData() []string {
|
||||
var badDomains []string
|
||||
listSimple := make(chan []string)
|
||||
listComplex := make(chan []string)
|
||||
listAdBlock := make(chan []string)
|
||||
|
||||
cfg.Log.Info("downloading blocklists")
|
||||
log.Info("downloading blocklists")
|
||||
// Get Simple Blocklists
|
||||
go func() {
|
||||
data := getData(cfg.ConfigFile.Sources.DomainListURLs)
|
||||
@@ -26,28 +28,39 @@ func getListData() []string {
|
||||
listComplex <- domains
|
||||
}()
|
||||
|
||||
// Get AdBlock Blocklists
|
||||
go func() {
|
||||
data := getData(cfg.ConfigFile.Sources.AdBlockURLs)
|
||||
domains := parseAdBlock(data)
|
||||
listAdBlock <- domains
|
||||
}()
|
||||
|
||||
// Wait for all downloads to finish
|
||||
var (
|
||||
simple, complex []string
|
||||
simpleFinished, complexFinished bool
|
||||
simple, complex, adblock []string
|
||||
simpleFinished, complexFinished, adBlockFinished bool
|
||||
)
|
||||
|
||||
for {
|
||||
select {
|
||||
case simple = <-listSimple:
|
||||
log.Info("all simple lists downloaded")
|
||||
simpleFinished = true
|
||||
cfg.Log.Info("all simple lists downloaded")
|
||||
case complex = <-listComplex:
|
||||
cfg.Log.Info("all complex lists downloaded")
|
||||
log.Info("all complex lists downloaded")
|
||||
complexFinished = true
|
||||
case adblock = <-listAdBlock:
|
||||
log.Info("all adblock lists downloaded")
|
||||
adBlockFinished = true
|
||||
default:
|
||||
time.Sleep(time.Millisecond * 100)
|
||||
}
|
||||
|
||||
if simpleFinished && complexFinished {
|
||||
if simpleFinished && complexFinished && adBlockFinished {
|
||||
badDomains = append(badDomains, simple...)
|
||||
badDomains = append(badDomains, complex...)
|
||||
cfg.Log.Info("domains retrieved", "hosts", len(badDomains))
|
||||
badDomains = append(badDomains, adblock...)
|
||||
log.Info("domains retrieved", "hosts", len(badDomains))
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -62,11 +75,11 @@ func getData(urls []string) []byte {
|
||||
listData := make([]byte, 0, len(urls)+1)
|
||||
|
||||
for _, u := range urls {
|
||||
cfg.Log.Debug("downloading", "url", u)
|
||||
log.Debug("downloading", "url", u)
|
||||
c := httpclient.DefaultClient()
|
||||
data, err := c.Get(u)
|
||||
if err != nil {
|
||||
cfg.Log.Error("unable to get remote content", "error", err, "url", err)
|
||||
log.Error("unable to get remote content", "error", err, "url", err)
|
||||
}
|
||||
listData = append(listData, data...)
|
||||
// add newline to the end of data, you know, for funzies
|
||||
|
@@ -1,8 +1,11 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"pihole-blocklist/bind/internal/config"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/config"
|
||||
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/log"
|
||||
)
|
||||
|
||||
var cfg config.Config
|
||||
@@ -11,7 +14,12 @@ func main() {
|
||||
// Blocklist Collection
|
||||
// https://firebog.net/
|
||||
|
||||
cfg = config.Init()
|
||||
var err error
|
||||
|
||||
if cfg, err = config.Init(); err != nil {
|
||||
log.Error("error initializing program", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Set the zone serial number
|
||||
cfg.ConfigFile.ZoneConfig.Serial = time.Now().In(cfg.TZLocal).Format("0601021504")
|
||||
@@ -24,6 +32,6 @@ func main() {
|
||||
|
||||
// write file
|
||||
if err := buildBindResponsePolicyFile(); err != nil {
|
||||
cfg.Log.Error("unable to write file", "error", err, "path", cfg.BindOutputFileName)
|
||||
log.Error("unable to write file", "error", err, "path", cfg.BindOutputFileName)
|
||||
}
|
||||
}
|
||||
|
53
cmd/bind/parsing-adblock.go
Normal file
53
cmd/bind/parsing-adblock.go
Normal file
@@ -0,0 +1,53 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"regexp"
|
||||
|
||||
"github.com/asaskevich/govalidator"
|
||||
|
||||
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/log"
|
||||
)
|
||||
|
||||
func parseAdBlock(data []byte) []string {
|
||||
var domains []string
|
||||
|
||||
// convert data to reader for line-by-line reading
|
||||
r := bytes.NewReader(data)
|
||||
|
||||
// process combined files line-by-line
|
||||
scanner := bufio.NewScanner(r)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
|
||||
// skip lines with the AdBlock header
|
||||
if regexp.MustCompile(`^\[Adblock Plus\]`).MatchString(line) {
|
||||
continue
|
||||
}
|
||||
|
||||
// skip lines where the first non-whitespace character is '#' or '//'
|
||||
if regexp.MustCompile(`^(\s+)?(#|\/\/|\!)`).MatchString(line) {
|
||||
continue
|
||||
}
|
||||
|
||||
// skip lines with no characters and/or whitespace only
|
||||
if regexp.MustCompile(`^(\s+)?$`).MatchString(line) {
|
||||
continue
|
||||
}
|
||||
|
||||
// remove line header
|
||||
d := regexp.MustCompile(`^\|\|`).ReplaceAllString(line, "")
|
||||
|
||||
// remove line footer
|
||||
d = regexp.MustCompile(`\^$`).ReplaceAllString(d, "")
|
||||
|
||||
if govalidator.IsDNSName(d) {
|
||||
domains = append(domains, d)
|
||||
} else {
|
||||
log.Debug("host invalid", "host", d)
|
||||
}
|
||||
}
|
||||
|
||||
return domains
|
||||
}
|
@@ -7,6 +7,8 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/asaskevich/govalidator"
|
||||
|
||||
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/log"
|
||||
)
|
||||
|
||||
func parseComplex(data []byte) []string {
|
||||
@@ -38,7 +40,7 @@ func parseComplex(data []byte) []string {
|
||||
if govalidator.IsDNSName(lineItems[1]) {
|
||||
domains = append(domains, lineItems[1])
|
||||
} else {
|
||||
cfg.Log.Debug("host invalid", "host", lineItems[0])
|
||||
log.Debug("host invalid", "host", lineItems[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@@ -7,6 +7,8 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/asaskevich/govalidator"
|
||||
|
||||
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/log"
|
||||
)
|
||||
|
||||
func parseSimple(data []byte) []string {
|
||||
@@ -38,7 +40,7 @@ func parseSimple(data []byte) []string {
|
||||
if govalidator.IsDNSName(lineItems[0]) {
|
||||
domains = append(domains, lineItems[0])
|
||||
} else {
|
||||
cfg.Log.Debug("host invalid", "host", lineItems[0])
|
||||
log.Debug("host invalid", "host", lineItems[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
Reference in New Issue
Block a user