nhyatt/bind-response-policy-zone-creator
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

some library updates and adds support for adblock lists

Browse Source
This commit is contained in:
Hyatt
2025-05-02 20:57:03 -05:00
parent 03b1cc13ee
commit ce4b4a11ff
19 changed files with 712 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
cmd/bind/build-bind.go
View File

@@ -5,8 +5,10 @@ import (
"text/template"
"pihole-blocklist/bind/assets"
"pihole-blocklist/bind/internal/config"
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/assets"
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/common"
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/log"
)
func buildBindResponsePolicyFile() error {
@@ -25,11 +27,11 @@ func buildBindResponsePolicyFile() error {
return err
}
bytesWritten, err := config.WriteFile(cfg.BindOutputFileName, output.Bytes())
bytesWritten, err := common.WriteFile(cfg.BindOutputFileName, output.Bytes())
if err != nil {
return err
}
cfg.Log.Debug("file created", "file", cfg.BindOutputFileName, "bytes", bytesWritten)
log.Debug("file created", "file", cfg.BindOutputFileName, "bytes", bytesWritten)
return nil
}

14
cmd/bind/cleanup.go
View File

@@ -3,6 +3,8 @@ package main
import (
"regexp"
"sort"
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/log"
)
func cleanBadDomains(domains []string) []string {
@@ -17,7 +19,7 @@ func cleanBadDomains(domains []string) []string {
}
}
domains = list
cfg.Log.Info("hosts removed from blocklist", "reason", "duplicate", "hosts", total-len(domains))
log.Info("hosts removed from blocklist", "reason", "duplicate", "hosts", total-len(domains))
// remove hosts that are too long
total = len(domains)
@@ -29,7 +31,7 @@ func cleanBadDomains(domains []string) []string {
list = append(list, blocklistItem)
}
domains = list
cfg.Log.Info("hosts removed from blocklist", "reason", "too many characters", "hosts", total-len(domains))
log.Info("hosts removed from blocklist", "reason", "too many characters", "hosts", total-len(domains))
// remove allow-listed matches
total = len(domains)
@@ -39,7 +41,7 @@ func cleanBadDomains(domains []string) []string {
for _, allowedItem := range cfg.ConfigFile.AllowLists {
_, err := regexp.Compile(allowedItem)
if err != nil {
cfg.Log.Error("unable to parse allow list item", "error", err, "regex", allowedItem)
log.Error("unable to parse allow list item", "error", err, "regex", allowedItem)
continue
}
goodAllowedItemList = append(goodAllowedItemList, allowedItem)
@@ -50,7 +52,7 @@ func cleanBadDomains(domains []string) []string {
addEntry := true
for _, allowedItem := range goodAllowedItemList {
if regexp.MustCompile(allowedItem).MatchString(v) {
cfg.Log.Debug("hosts removed from blocklist", "reason", "allowed host", "match string", allowedItem, "host", v)
log.Debug("hosts removed from blocklist", "reason", "allowed host", "match string", allowedItem, "host", v)
addEntry = false
}
}
@@ -59,9 +61,9 @@ func cleanBadDomains(domains []string) []string {
}
}
domains = list
cfg.Log.Info("hosts removed from blocklist", "hosts", total-len(domains))
log.Info("hosts removed from blocklist", "hosts", total-len(domains))
cfg.Log.Info("total domains in list", "hosts", len(domains))
log.Info("total domains in list", "hosts", len(domains))
sort.Strings(domains)
return domains
}

33
cmd/bind/get-remote-data.go
View File

@@ -3,15 +3,17 @@ package main
import (
"time"
"pihole-blocklist/bind/internal/httpclient"
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/httpclient"
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/log"
)
func getListData() []string {
var badDomains []string
listSimple := make(chan []string)
listComplex := make(chan []string)
listAdBlock := make(chan []string)
cfg.Log.Info("downloading blocklists")
log.Info("downloading blocklists")
// Get Simple Blocklists
go func() {
data := getData(cfg.ConfigFile.Sources.DomainListURLs)
@@ -26,28 +28,39 @@ func getListData() []string {
listComplex <- domains
}()
// Get AdBlock Blocklists
go func() {
data := getData(cfg.ConfigFile.Sources.AdBlockURLs)
domains := parseAdBlock(data)
listAdBlock <- domains
}()
// Wait for all downloads to finish
var (
simple, complex []string
simpleFinished, complexFinished bool
simple, complex, adblock []string
simpleFinished, complexFinished, adBlockFinished bool
)
for {
select {
case simple = <-listSimple:
log.Info("all simple lists downloaded")
simpleFinished = true
cfg.Log.Info("all simple lists downloaded")
case complex = <-listComplex:
cfg.Log.Info("all complex lists downloaded")
log.Info("all complex lists downloaded")
complexFinished = true
case adblock = <-listAdBlock:
log.Info("all adblock lists downloaded")
adBlockFinished = true
default:
time.Sleep(time.Millisecond * 100)
}
if simpleFinished && complexFinished {
if simpleFinished && complexFinished && adBlockFinished {
badDomains = append(badDomains, simple...)
badDomains = append(badDomains, complex...)
cfg.Log.Info("domains retrieved", "hosts", len(badDomains))
badDomains = append(badDomains, adblock...)
log.Info("domains retrieved", "hosts", len(badDomains))
break
}
}
@@ -62,11 +75,11 @@ func getData(urls []string) []byte {
listData := make([]byte, 0, len(urls)+1)
for _, u := range urls {
cfg.Log.Debug("downloading", "url", u)
log.Debug("downloading", "url", u)
c := httpclient.DefaultClient()
data, err := c.Get(u)
if err != nil {
cfg.Log.Error("unable to get remote content", "error", err, "url", err)
log.Error("unable to get remote content", "error", err, "url", err)
}
listData = append(listData, data...)
// add newline to the end of data, you know, for funzies

14
cmd/bind/main.go
View File

@@ -1,8 +1,11 @@
package main
import (
"pihole-blocklist/bind/internal/config"
"os"
"time"
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/config"
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/log"
)
var cfg config.Config
@@ -11,7 +14,12 @@ func main() {
// Blocklist Collection
// https://firebog.net/
cfg = config.Init()
var err error
if cfg, err = config.Init(); err != nil {
log.Error("error initializing program", "error", err)
os.Exit(1)
}
// Set the zone serial number
cfg.ConfigFile.ZoneConfig.Serial = time.Now().In(cfg.TZLocal).Format("0601021504")
@@ -24,6 +32,6 @@ func main() {
// write file
if err := buildBindResponsePolicyFile(); err != nil {
cfg.Log.Error("unable to write file", "error", err, "path", cfg.BindOutputFileName)
log.Error("unable to write file", "error", err, "path", cfg.BindOutputFileName)
}
}

53
cmd/bind/parsing-adblock.go Normal file
View File

@@ -0,0 +1,53 @@
package main
import (
"bufio"
"bytes"
"regexp"
"github.com/asaskevich/govalidator"
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/log"
)
func parseAdBlock(data []byte) []string {
var domains []string
// convert data to reader for line-by-line reading
r := bytes.NewReader(data)
// process combined files line-by-line
scanner := bufio.NewScanner(r)
for scanner.Scan() {
line := scanner.Text()
// skip lines with the AdBlock header
if regexp.MustCompile(`^\[Adblock Plus\]`).MatchString(line) {
continue
}
// skip lines where the first non-whitespace character is '#' or '//'
if regexp.MustCompile(`^(\s+)?(#|\/\/|\!)`).MatchString(line) {
continue
}
// skip lines with no characters and/or whitespace only
if regexp.MustCompile(`^(\s+)?$`).MatchString(line) {
continue
}
// remove line header
d := regexp.MustCompile(`^\|\|`).ReplaceAllString(line, "")
// remove line footer
d = regexp.MustCompile(`\^$`).ReplaceAllString(d, "")
if govalidator.IsDNSName(d) {
domains = append(domains, d)
} else {
log.Debug("host invalid", "host", d)
}
}
return domains
}

4
cmd/bind/parsing-complex.go
View File

@@ -7,6 +7,8 @@ import (
"strings"
"github.com/asaskevich/govalidator"
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/log"
)
func parseComplex(data []byte) []string {
@@ -38,7 +40,7 @@ func parseComplex(data []byte) []string {
if govalidator.IsDNSName(lineItems[1]) {
domains = append(domains, lineItems[1])
} else {
cfg.Log.Debug("host invalid", "host", lineItems[0])
log.Debug("host invalid", "host", lineItems[0])
}
}
}

4
cmd/bind/parsing-simple.go
View File

@@ -7,6 +7,8 @@ import (
"strings"
"github.com/asaskevich/govalidator"
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/log"
)
func parseSimple(data []byte) []string {
@@ -38,7 +40,7 @@ func parseSimple(data []byte) []string {
if govalidator.IsDNSName(lineItems[0]) {
domains = append(domains, lineItems[0])
} else {
cfg.Log.Debug("host invalid", "host", lineItems[0])
log.Debug("host invalid", "host", lineItems[0])
}
}
}