nhyatt/bind-response-policy-zone-creator
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

multiple updates

Browse Source
This commit is contained in:
Hyatt
2023-12-09 14:21:28 -06:00
parent a11f92745d
commit 881c11b910
23 changed files with 721 additions and 433 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
cmd/bind/build-bind.go
View File

@@ -2,64 +2,34 @@ package main
import (
"bytes"
"log"
"os"
"time"
"text/template"
"pihole-blocklist/bind/assets"
"pihole-blocklist/bind/internal/config"
)
func buildBindResponsePolicyFile() {
defer timeTrack(time.Now(), "buildBindResponsePolicyFile")
func buildBindResponsePolicyFile() error {
var (
output bytes.Buffer
)
outputTemplate := `{{- $domain := .Domain -}}
$TTL {{ or .TTL "1h" }}
@ IN SOA {{ $domain }}. {{ or .Email "domain-admin" }}. (
{{ or .Serial "0000000000" }} ; Serial
{{ or .Refresh "1h" }} ; Refresh
{{ or .Retry "30m" }} ; Retry
{{ or .Expire "1w" }} ; Expire
{{ or .Minimum "1h" }} ; Minimum
)
outputTemplate := assets.BindRecord
;
; Name Servers
;
{{- range .NameServers }}
IN NS {{ . }}.
{{- end }}
;
; Addresses
;
{{- range .BlockedDomains }}
{{ . }} IN CNAME blocked.{{ $domain }}.
{{- end }}
`
t, err := template.New("response-policy-zone").Parse(outputTemplate)
t, err := template.New("response-policy-zone").Parse(string(outputTemplate))
if err != nil {
log.Fatalf("[FATAL] Unable to parse template (%s): %v\n", "response-policy-zone", err)
return err
}
if err := t.Execute(&output, config.Config.ZoneConfig); err != nil {
log.Fatalf("[FATAL] Unable to generate template output: %v\n", err)
if err := t.Execute(&output, cfg.ConfigFile.ZoneConfig); err != nil {
return err
}
fileWriter, err := os.Create(config.BindOutputFileName)
bytesWritten, err := config.WriteFile(cfg.BindOutputFileName, output.Bytes())
if err != nil {
log.Fatalf("[FATAL] Unable to open file (%s) for writing: %v", config.BindOutputFileName, err)
}
defer fileWriter.Close()
bytesWritten, err := fileWriter.Write(output.Bytes())
if err != nil {
log.Fatalf("[FATAL] Unable to write to file (%s): %v", config.BindOutputFileName, err)
return err
}
log.Printf("[DEBUG] Wrote %d bytes to %s.\n", bytesWritten, config.BindOutputFileName)
cfg.Log.Debug("file created", "file", cfg.BindOutputFileName, "bytes", bytesWritten)
return nil
}

20
cmd/bind/cleanup.go
View File

@@ -1,15 +1,11 @@
package main
import (
"log"
"regexp"
"sort"
"time"
)
func cleanBadDomains(domains []string) []string {
defer timeTrack(time.Now(), "cleanBadDomains")
// remove duplicates
total := len(domains)
all := make(map[string]bool)
@@ -21,7 +17,7 @@ func cleanBadDomains(domains []string) []string {
}
}
domains = list
log.Printf("[INFO] Duplicate items removed: %d\n", total-len(domains))
cfg.Log.Info("hosts removed from blocklist", "reason", "duplicate", "hosts", total-len(domains))
// remove hosts that are too long
total = len(domains)
@@ -33,18 +29,18 @@ func cleanBadDomains(domains []string) []string {
list = append(list, blocklistItem)
}
domains = list
log.Printf("[INFO] Hosts with too many characters removed: %d\n", total-len(domains))
cfg.Log.Info("hosts removed from blocklist", "reason", "too many characters", "hosts", total-len(domains))
// remove allow-listed matches
total = len(domains)
// filter out bad regex
goodAllowedItemList := []string{}
for _, allowedItem := range config.Config.AllowLists {
for _, allowedItem := range cfg.ConfigFile.AllowLists {
_, err := regexp.Compile(allowedItem)
if err != nil {
log.Printf("[ERROR] Allow list item (%s) is not valid regex: %v\n", allowedItem, err)
break
cfg.Log.Error("unable to parse allow list item", "error", err, "regex", allowedItem)
continue
}
goodAllowedItemList = append(goodAllowedItemList, allowedItem)
}
@@ -54,7 +50,7 @@ func cleanBadDomains(domains []string) []string {
addEntry := true
for _, allowedItem := range goodAllowedItemList {
if regexp.MustCompile(allowedItem).MatchString(v) {
log.Printf("[DEBUG] Removing allowed matching item: %s\n", v)
cfg.Log.Debug("hosts removed from blocklist", "reason", "allowed host", "match string", allowedItem, "host", v)
addEntry = false
}
}
@@ -63,9 +59,9 @@ func cleanBadDomains(domains []string) []string {
}
}
domains = list
log.Printf("[INFO] Allowed hosts removed: %d\n", total-len(domains))
cfg.Log.Info("hosts removed from blocklist", "hosts", total-len(domains))
log.Printf("[INFO] Total domains in list at end: %d.\n", len(domains))
cfg.Log.Info("total domains in list", "hosts", len(domains))
sort.Strings(domains)
return domains
}

103
cmd/bind/config.go
View File

@@ -1,103 +0,0 @@
package main
import (
"os"
"time"
"github.com/hashicorp/logutils"
)
type configStructure struct {
// time configuration
TimeFormat string
TimeZone *time.Location
TimeZoneUTC *time.Location
// logging
Log *logutils.LevelFilter
// HTTP Client timeout configurations
HTTPClientRequestTimeout int
HTTPClientConnectTimeout int
HTTPClientTLSHandshakeTimeout int
HTTPClientIdleTimeout int
// Output Filename
BindOutputFileName string
// Config
ConfigFileLocation string
Config configFileStruct
}
type configFileStruct struct {
ZoneConfig struct {
TTL string `yaml:"timeToLive"`
Domain string `yaml:"baseDomain"`
Email string `yaml:"emailAddress"`
Serial string `yaml:"zoneSerialNumber"`
Refresh string `yaml:"zoneRefresh"`
Retry string `yaml:"zoneRetry"`
Expire string `yaml:"zoneExpire"`
Minimum string `yaml:"zoneMinimum"`
NameServers []string `yaml:"nameServers"`
BlockedDomains []string `yaml:"blockedDomains"`
} `yaml:"zoneConfig"`
Sources struct {
HostFileURLs []string `yaml:"hostFileURLs"`
DomainListURLs []string `yaml:"domainListURLs"`
} `yaml:"sources"`
AllowLists []string `yaml:"allowList"`
DenyList []string `yaml:"denyList"`
}
var config = configStructure{
TimeFormat: "2006-01-02 15:04:05",
Log: &logutils.LevelFilter{
Levels: []logutils.LogLevel{"TRACE", "DEBUG", "INFO", "WARNING", "ERROR"},
Writer: os.Stderr,
},
// Nice blocklist location: https://firebog.net/
// Default Blocklist
Config: configFileStruct{
Sources: struct {
HostFileURLs []string `yaml:"hostFileURLs"`
DomainListURLs []string `yaml:"domainListURLs"`
}{
HostFileURLs: []string{
//"https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts",
//"http://sysctl.org/cameleon/hosts",
//"https://raw.githubusercontent.com/DandelionSprout/adfilt/master/Alternate%20versions%20Anti-Malware%20List/AntiMalwareHosts.txt",
//"https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Risk/hosts",
},
DomainListURLs: []string{
//"https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt",
//"https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt",
//"https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt",
//"https://v.firebog.net/hosts/Prigent-Crypto.txt",
//"https://phishing.army/download/phishing_army_blocklist_extended.txt",
//"https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-malware.txt",
//"https://raw.githubusercontent.com/Spam404/lists/master/main-blacklist.txt",
//"https://dbl.oisd.nl/",
//"https://osint.digitalside.it/Threat-Intel/lists/latestdomains.txt",
},
},
AllowLists: []string{
// localhosts included in blocklists for some reason
`localhost`,
`localhost.localdomain`,
`local`,
`broadcasthost`,
`localhost`,
`ip6-localhost`,
`ip6-loopback`,
`localhost`,
`ip6-localnet`,
`ip6-mcastprefix`,
`ip6-allnodes`,
`ip6-allrouters`,
`ip6-allhosts`,
},
},
}

28
cmd/bind/get-remote-data.go
View File

@@ -1,29 +1,27 @@
package main
import (
"log"
"pihole-blocklist/v2/internal/httpclient"
"time"
"pihole-blocklist/bind/internal/httpclient"
)
func getListData() []string {
defer timeTrack(time.Now(), "getListData")
var badDomains []string
listSimple := make(chan []string)
listComplex := make(chan []string)
log.Printf("[INFO] Downloading blocklists\n")
cfg.Log.Info("downloading blocklists")
// Get Simple Blocklists
go func() {
data := getData(config.Config.Sources.DomainListURLs)
data := getData(cfg.ConfigFile.Sources.DomainListURLs)
domains := parseSimple(data)
listSimple <- domains
}()
// Get Host File Blocklists
go func() {
data := getData(config.Config.Sources.HostFileURLs)
data := getData(cfg.ConfigFile.Sources.HostFileURLs)
domains := parseComplex(data)
listComplex <- domains
}()
@@ -38,9 +36,9 @@ func getListData() []string {
select {
case simple = <-listSimple:
simpleFinished = true
log.Printf("[INFO] All simple lists have been retrieved.\n")
cfg.Log.Info("all simple lists downloaded")
case complex = <-listComplex:
log.Printf("[INFO] All complex lists have been retrieved.\n")
cfg.Log.Info("all complex lists downloaded")
complexFinished = true
default:
time.Sleep(time.Millisecond * 100)
@@ -49,28 +47,26 @@ func getListData() []string {
if simpleFinished && complexFinished {
badDomains = append(badDomains, simple...)
badDomains = append(badDomains, complex...)
log.Printf("[INFO] Number of domains detected: %d\n", len(badDomains))
cfg.Log.Info("domains retrieved", "hosts", len(badDomains))
break
}
}
// append deny list items to list of blocked domains
badDomains = append(badDomains, config.Config.DenyList...)
badDomains = append(badDomains, cfg.ConfigFile.DenyList...)
return badDomains
}
func getData(urls []string) []byte {
defer timeTrack(time.Now(), "getData")
var listData []byte
listData := make([]byte, 0, len(urls)+1)
for _, u := range urls {
log.Printf("[TRACE] Downloading URL: %s\n", u)
cfg.Log.Debug("downloading", "url", u)
c := httpclient.DefaultClient()
data, err := c.Get(u)
if err != nil {
log.Printf("[ERROR] Unable to get remote content from URL (%s): %v", u, err)
cfg.Log.Error("unable to get remote content", "error", err, "url", err)
}
listData = append(listData, data...)
// add newline to the end of data, you know, for funzies

212
cmd/bind/init.go
View File

@@ -1,212 +0,0 @@
package main
import (
"flag"
"io/ioutil"
"log"
"os"
"strconv"
"strings"
"time"
"github.com/hashicorp/logutils"
"gopkg.in/yaml.v3"
)
// getEnvString returns string from environment variable
func getEnvString(env, def string) (val string) { //nolint:deadcode
defer timeTrack(time.Now(), "getEnvString")
val = os.Getenv(env)
if val == "" {
return def
}
return
}
// getEnvInt returns int from environment variable
func getEnvInt(env string, def int) (ret int) {
defer timeTrack(time.Now(), "getEnvInt")
val := os.Getenv(env)
if val == "" {
return def
}
ret, err := strconv.Atoi(val)
if err != nil {
log.Fatalf("[ERROR] Environment variable is not numeric: %v\n", env)
}
return
}
func initialize() {
defer timeTrack(time.Now(), "initialize")
config.TimeZone, _ = time.LoadLocation("America/Chicago")
config.TimeZoneUTC, _ = time.LoadLocation("UTC")
// read command line options
var (
logLevel int
ns1, ns2 string
)
// log configuration
flag.IntVar(&logLevel,
"log",
getEnvInt("LOG_LEVEL", 50),
"(LOG_LEVEL)\nlog level")
// http client configuration
flag.IntVar(&config.HTTPClientRequestTimeout,
"client-req-to",
getEnvInt("HTTP_CLIENT_REQUEST_TIMEOUT", 60),
"(HTTP_CLIENT_REQUEST_TIMEOUT)\ntime in seconds for the internal http client to complete a request")
flag.IntVar(&config.HTTPClientConnectTimeout,
"client-con-to",
getEnvInt("HTTP_CLIENT_CONNECT_TIMEOUT", 5),
"(HTTP_CLIENT_CONNECT_TIMEOUT)\ntime in seconds for the internal http client connection timeout")
flag.IntVar(&config.HTTPClientTLSHandshakeTimeout,
"client-tls-to",
getEnvInt("HTTP_CLIENT_TLS_TIMEOUT", 5),
"(HTTP_CLIENT_TLS_TIMEOUT)\ntime in seconds for the internal http client to complete a tls handshake")
flag.IntVar(&config.HTTPClientIdleTimeout,
"client-idle-to",
getEnvInt("HTTP_CLIENT_IDLE_TIMEOUT", 5),
"(HTTP_CLIENT_IDLE_TIMEOUT)\ntime in seconds that the internal http client will keep a connection open when idle")
// Bind Config
flag.StringVar(&config.Config.ZoneConfig.TTL,
"bind-ttl",
getEnvString("TTL", "1h"),
"(TTL)\nBind zone time to live")
flag.StringVar(&config.Config.ZoneConfig.Domain,
"bind-domain",
getEnvString("DOMAIN", "example.com"),
"(DOMAIN)\nBind zone base domain")
flag.StringVar(&config.Config.ZoneConfig.Email,
"bind-email",
getEnvString("EMAIL", "domain-admin@example.com"),
"(EMAIL)\nBind zone authority e-mail address")
flag.StringVar(&config.Config.ZoneConfig.Serial,
"bind-timestamp",
getEnvString("TIMESTAMP", time.Now().In(config.TimeZone).Format("0601021504")),
"(TIMESTAMP)\nBind zone serial number")
flag.StringVar(&config.Config.ZoneConfig.Refresh,
"bind-refresh",
getEnvString("REFRESH", "1h"),
"(REFRESH)\nBind zone refresh time")
flag.StringVar(&config.Config.ZoneConfig.Retry,
"bind-retry",
getEnvString("RETRY", "30m"),
"(RETRY)\nBind zone retry time")
flag.StringVar(&config.Config.ZoneConfig.Expire,
"bind-expire",
getEnvString("EXPIRE", "1w"),
"(EXPIRE)\nBind zone expire time")
flag.StringVar(&config.Config.ZoneConfig.Minimum,
"bind-minimum",
getEnvString("MINIMUM", "1h"),
"(MINIMUM)\nBind zone minimum time")
flag.StringVar(&ns1,
"bind-ns1",
getEnvString("NS1", ""),
"(NS1)\nBind zone primary name-server")
flag.StringVar(&ns2,
"bind-ns2",
getEnvString("NS2", ""),
"(NS2)\nBind zone secondary name-server")
// output file
flag.StringVar(&config.BindOutputFileName,
"output",
getEnvString("OUTPUT", "./response-policy.bind"),
"(FILENAME)\nWrite local file to filename")
flag.StringVar(&config.ConfigFileLocation,
"config-file",
getEnvString("CONFIG_FILE", ""),
"(CONFIG_FILE)\nRead configuration from file")
flag.Parse()
// set logging level
switch {
case logLevel <= 20:
config.Log.SetMinLevel(logutils.LogLevel("ERROR"))
case logLevel > 20 && logLevel <= 40:
config.Log.SetMinLevel(logutils.LogLevel("WARNING"))
case logLevel > 40 && logLevel <= 60:
config.Log.SetMinLevel(logutils.LogLevel("INFO"))
case logLevel > 60 && logLevel <= 80:
config.Log.SetMinLevel(logutils.LogLevel("DEBUG"))
case logLevel > 80:
config.Log.SetMinLevel(logutils.LogLevel("TRACE"))
}
log.SetOutput(config.Log)
// print current configuration
log.Printf("[DEBUG] configuration value set: LOG_LEVEL = %v\n", strconv.Itoa(logLevel))
log.Printf("[DEBUG] configuration value set: HTTP_CLIENT_REQUEST_TIMEOUT = %v\n", strconv.Itoa(config.HTTPClientRequestTimeout))
log.Printf("[DEBUG] configuration value set: HTTP_CLIENT_CONNECT_TIMEOUT = %v\n", strconv.Itoa(config.HTTPClientConnectTimeout))
log.Printf("[DEBUG] configuration value set: HTTP_CLIENT_TLS_TIMEOUT = %v\n", strconv.Itoa(config.HTTPClientTLSHandshakeTimeout))
log.Printf("[DEBUG] configuration value set: HTTP_CLIENT_IDLE_TIMEOUT = %v\n", strconv.Itoa(config.HTTPClientIdleTimeout))
log.Printf("[DEBUG] configuration value set: TTL = %v\n", config.Config.ZoneConfig.TTL)
log.Printf("[DEBUG] configuration value set: DOMAIN = %v\n", config.Config.ZoneConfig.Domain)
log.Printf("[DEBUG] configuration value set: EMAIL = %v\n", config.Config.ZoneConfig.Email)
log.Printf("[DEBUG] configuration value set: TIMESTAMP = %v\n", config.Config.ZoneConfig.Serial)
log.Printf("[DEBUG] configuration value set: REFRESH = %v\n", config.Config.ZoneConfig.Refresh)
log.Printf("[DEBUG] configuration value set: RETRY = %v\n", config.Config.ZoneConfig.Retry)
log.Printf("[DEBUG] configuration value set: EXPIRE = %v\n", config.Config.ZoneConfig.Expire)
log.Printf("[DEBUG] configuration value set: MINIMUM = %v\n", config.Config.ZoneConfig.Minimum)
log.Printf("[DEBUG] configuration value set: NS1 = %v\n", ns1)
log.Printf("[DEBUG] configuration value set: NS2 = %v\n", ns2)
log.Printf("[DEBUG] configuration value set: CONFIG_FILE = %v\n", config.ConfigFileLocation)
// read config file
var err error
if config.ConfigFileLocation != "" {
if config.Config, err = readConfigFile(config.ConfigFileLocation); err != nil {
log.Fatalf("[FATAL] Invalid config file: %v\n", err)
}
if config.Config.ZoneConfig.Serial == "" {
config.Config.ZoneConfig.Serial = time.Now().In(config.TimeZone).Format("0601021504")
}
}
// set bind-config nameservers
if ns1 != "" {
config.Config.ZoneConfig.NameServers = append(config.Config.ZoneConfig.NameServers, ns1)
}
if ns2 != "" {
config.Config.ZoneConfig.NameServers = append(config.Config.ZoneConfig.NameServers, ns2)
}
if len(config.Config.ZoneConfig.NameServers) == 0 {
log.Printf("[ERROR] A primary name-server must be identified.")
flag.PrintDefaults()
os.Exit(1)
}
// bind does not use "@", so we convert it to a "."
config.Config.ZoneConfig.Email = strings.Replace(config.Config.ZoneConfig.Email, "@", ".", -1)
log.Printf("[DEBUG] Initialization Complete\n")
}
func readConfigFile(configFileLocation string) (configFileStruct, error) {
defer timeTrack(time.Now(), "readConfigFile")
var output configFileStruct
rd, err := ioutil.ReadFile(configFileLocation)
if err != nil {
return output, err
}
if err := yaml.Unmarshal(rd, &output); err != nil {
return output, err
}
return output, nil
}

19
cmd/bind/main.go
View File

@@ -1,13 +1,26 @@
package main
import (
"pihole-blocklist/bind/internal/config"
"time"
)
var cfg config.Config
func main() {
initialize()
cfg = config.Init()
// Set the zone serial number
cfg.ConfigFile.ZoneConfig.Serial = time.Now().In(cfg.TZLocal).Format("0601021504")
// get remote URL data
badDomains := getListData()
// clean-up
config.Config.ZoneConfig.BlockedDomains = cleanBadDomains(badDomains)
cfg.ConfigFile.ZoneConfig.BlockedDomains = cleanBadDomains(badDomains)
buildBindResponsePolicyFile()
// write file
if err := buildBindResponsePolicyFile(); err != nil {
cfg.Log.Error("unable to write file", "error", err, "path", cfg.BindOutputFileName)
}
}

6
cmd/bind/parsing-complex.go
View File

@@ -3,17 +3,13 @@ package main
import (
"bufio"
"bytes"
"log"
"regexp"
"strings"
"time"
"github.com/asaskevich/govalidator"
)
func parseComplex(data []byte) []string {
defer timeTrack(time.Now(), "parseComplex")
var domains []string
// convert data to reader for line-by-line reading
@@ -42,7 +38,7 @@ func parseComplex(data []byte) []string {
if govalidator.IsDNSName(lineItems[1]) {
domains = append(domains, lineItems[1])
} else {
log.Printf("[TRACE] Domain is not valid: %s\n", lineItems[0])
cfg.Log.Debug("host invalid", "host", lineItems[0])
}
}
}

6
cmd/bind/parsing-simple.go
View File

@@ -3,17 +3,13 @@ package main
import (
"bufio"
"bytes"
"log"
"regexp"
"strings"
"time"
"github.com/asaskevich/govalidator"
)
func parseSimple(data []byte) []string {
defer timeTrack(time.Now(), "parseSimple")
var domains []string
// convert data to reader for line-by-line reading
@@ -42,7 +38,7 @@ func parseSimple(data []byte) []string {
if govalidator.IsDNSName(lineItems[0]) {
domains = append(domains, lineItems[0])
} else {
log.Printf("[TRACE] Domain is not valid: %s\n", lineItems[0])
cfg.Log.Debug("host invalid", "host", lineItems[0])
}
}
}

16
cmd/bind/supporting-functions.go
View File

@@ -1,16 +0,0 @@
package main
import (
"log"
"time"
)
func timeTrack(start time.Time, name string) {
elapsed := time.Since(start)
log.Printf("[DEBUG] Function %s took %s\n", name, elapsed)
}
func removeStringFromSlice(s []string, i int) []string {
s[i] = s[len(s)-1]
return s[:len(s)-1]
}