initial commit
This commit is contained in:
commit
89e13a7114
GPG Key ID:
C50D0BBB5BC40BEA
61
cmd/bind/build-bind.go
Normal file
61
cmd/bind/build-bind.go
Normal file
@ -0,0 +1,61 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"text/template"
|
||||
)
|
||||
|
||||
func buildBindResponsePolicyFile() {
|
||||
var (
|
||||
output bytes.Buffer
|
||||
)
|
||||
|
||||
outputTemplate := `{{- $domain := .Domain -}}
|
||||
$TTL {{ or .TTL "1h" }}
|
||||
@ IN SOA {{ $domain }}. {{ or .Email "domain-admin" }}. (
|
||||
{{ or .Timestamp "0000000000" }} ; Serial
|
||||
{{ or .Refresh "1h" }} ; Refresh
|
||||
{{ or .Retry "30m" }} ; Retry
|
||||
{{ or .Expire "1w" }} ; Expire
|
||||
{{ or .Minimum "1h" }} ; Minimum
|
||||
)
|
||||
|
||||
;
|
||||
; Name Servers
|
||||
;
|
||||
{{- range .NameServers }}
|
||||
IN NS {{ . }}.
|
||||
{{- end }}
|
||||
|
||||
;
|
||||
; Addresses
|
||||
;
|
||||
{{- range .BadDomains }}
|
||||
{{ . }} IN CNAME blocked.{{ $domain }}.
|
||||
{{- end }}`
|
||||
|
||||
t, err := template.New("response-policy-zone").Parse(outputTemplate)
|
||||
if err != nil {
|
||||
log.Fatalf("[FATAL] Unable to parse template (%s): %v\n", "response-policy-zone", err)
|
||||
}
|
||||
|
||||
if err := t.Execute(&output, config.NamedConfig); err != nil {
|
||||
log.Fatalf("[FATAL] Unable to generate template output: %v\n", err)
|
||||
}
|
||||
|
||||
fileWriter, err := os.Create(config.BindOutputFileName)
|
||||
if err != nil {
|
||||
log.Fatalf("[FATAL] Unable to open file (%s) for writing: %v", config.BindOutputFileName, err)
|
||||
}
|
||||
defer fileWriter.Close()
|
||||
|
||||
bytesWritten, err := fileWriter.Write(output.Bytes())
|
||||
if err != nil {
|
||||
log.Fatalf("[FATAL] Unable to write to file (%s): %v", config.BindOutputFileName, err)
|
||||
}
|
||||
|
||||
log.Printf("[DEBUG] Wrote %d bytes to %s.\n", bytesWritten, config.BindOutputFileName)
|
||||
}
|
||||
118
cmd/bind/config.go
Normal file
118
cmd/bind/config.go
Normal file
@ -0,0 +1,118 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/logutils"
|
||||
)
|
||||
|
||||
type configStructure struct {
|
||||
// time configuration
|
||||
TimeFormat string
|
||||
TimeZone *time.Location
|
||||
TimeZoneUTC *time.Location
|
||||
|
||||
// logging
|
||||
Log *logutils.LevelFilter
|
||||
|
||||
// HTTP Client timeout configurations
|
||||
HTTPClientRequestTimeout int
|
||||
HTTPClientConnectTimeout int
|
||||
HTTPClientTLSHandshakeTimeout int
|
||||
HTTPClientIdleTimeout int
|
||||
|
||||
// Download Sources
|
||||
URLBlocklistHostFiles []string
|
||||
URLBlocklistsSimple []string
|
||||
|
||||
// Allowlist (regex)
|
||||
DomainAllowlist []*regexp.Regexp
|
||||
|
||||
// Named Config Generator
|
||||
NamedConfig namedConfigStruct
|
||||
|
||||
// Output Filename
|
||||
BindOutputFileName string
|
||||
}
|
||||
|
||||
type namedConfigStruct struct {
|
||||
TTL string
|
||||
Domain string
|
||||
Email string
|
||||
Timestamp string
|
||||
Refresh string
|
||||
Retry string
|
||||
Expire string
|
||||
Minimum string
|
||||
NameServers []string
|
||||
BadDomains []string
|
||||
}
|
||||
|
||||
var config = configStructure{
|
||||
TimeFormat: "2006-01-02 15:04:05",
|
||||
Log: &logutils.LevelFilter{
|
||||
Levels: []logutils.LogLevel{"TRACE", "DEBUG", "INFO", "WARNING", "ERROR"},
|
||||
Writer: os.Stderr,
|
||||
},
|
||||
|
||||
// Nice blocklist location: https://firebog.net/
|
||||
// Default Blocklist
|
||||
URLBlocklistHostFiles: []string{
|
||||
"https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts",
|
||||
"http://sysctl.org/cameleon/hosts",
|
||||
"https://raw.githubusercontent.com/DandelionSprout/adfilt/master/Alternate%20versions%20Anti-Malware%20List/AntiMalwareHosts.txt",
|
||||
"https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Risk/hosts",
|
||||
},
|
||||
URLBlocklistsSimple: []string{
|
||||
"https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt",
|
||||
"https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt",
|
||||
"https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt",
|
||||
"https://v.firebog.net/hosts/Prigent-Crypto.txt",
|
||||
"https://phishing.army/download/phishing_army_blocklist_extended.txt",
|
||||
"https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-malware.txt",
|
||||
"https://raw.githubusercontent.com/Spam404/lists/master/main-blacklist.txt",
|
||||
"https://dbl.oisd.nl/",
|
||||
"https://osint.digitalside.it/Threat-Intel/lists/latestdomains.txt",
|
||||
|
||||
},
|
||||
|
||||
// default URL Allow hosts
|
||||
DomainAllowlist: []*regexp.Regexp{
|
||||
// localhosts included in blocklists for some reason
|
||||
regexp.MustCompile(`localhost`),
|
||||
regexp.MustCompile(`localhost.localdomain`),
|
||||
regexp.MustCompile(`local`),
|
||||
regexp.MustCompile(`broadcasthost`),
|
||||
regexp.MustCompile(`localhost`),
|
||||
regexp.MustCompile(`ip6-localhost`),
|
||||
regexp.MustCompile(`ip6-loopback`),
|
||||
regexp.MustCompile(`localhost`),
|
||||
regexp.MustCompile(`ip6-localnet`),
|
||||
regexp.MustCompile(`ip6-mcastprefix`),
|
||||
regexp.MustCompile(`ip6-allnodes`),
|
||||
regexp.MustCompile(`ip6-allrouters`),
|
||||
regexp.MustCompile(`ip6-allhosts`),
|
||||
// default allow hosts
|
||||
regexp.MustCompile(`(^|\.)` + `thepiratebay\.org`),
|
||||
regexp.MustCompile(`(^|\.)` + `sendgrid\.net`),
|
||||
regexp.MustCompile(`(^|\.)` + `googleadservices\.com`),
|
||||
regexp.MustCompile(`(^|\.)` + `doubleclick\.net`),
|
||||
regexp.MustCompile(`(^|\.)` + `sailthru\.com`),
|
||||
regexp.MustCompile(`(^|\.)` + `magiskmanager\.com`),
|
||||
regexp.MustCompile(`(^|\.)` + `apiservices\.krxd\.net`),
|
||||
regexp.MustCompile(`(^|\.)` + `logfiles\.zoom\.us`),
|
||||
regexp.MustCompile(`(^|\.)` + `logfiles-va\.zoom\.us`),
|
||||
regexp.MustCompile(`(^|\.)` + `nest\.com`),
|
||||
regexp.MustCompile(`(^|\.)` + `clients.\.google\.com`),
|
||||
regexp.MustCompile(`(^|\.)` + `login\.live\.com`),
|
||||
regexp.MustCompile(`(^|\.)` + `unagi\.amazon\.com`),
|
||||
regexp.MustCompile(`(^|\.)` + `unagi-na\.amazon\.com`),
|
||||
regexp.MustCompile(`(^|\.)` + `duckduckgo\.com`),
|
||||
regexp.MustCompile(`(^|\.)` + `msn\.com`),
|
||||
regexp.MustCompile(`(^|\.)` + `nexusrules\.officeapps\.live\.com`),
|
||||
regexp.MustCompile(`(^|\.)` + `playfabapi\.com`),
|
||||
regexp.MustCompile(`(^|\.)` + `vercel-dns\.com`),
|
||||
},
|
||||
}
|
||||
167
cmd/bind/init.go
Normal file
167
cmd/bind/init.go
Normal file
@ -0,0 +1,167 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/logutils"
|
||||
)
|
||||
|
||||
// getEnvString returns string from environment variable
|
||||
func getEnvString(env, def string) (val string) { //nolint:deadcode
|
||||
val = os.Getenv(env)
|
||||
|
||||
if val == "" {
|
||||
return def
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// getEnvInt returns int from environment variable
|
||||
func getEnvInt(env string, def int) (ret int) {
|
||||
val := os.Getenv(env)
|
||||
|
||||
if val == "" {
|
||||
return def
|
||||
}
|
||||
|
||||
ret, err := strconv.Atoi(val)
|
||||
if err != nil {
|
||||
log.Fatalf("[ERROR] Environment variable is not numeric: %v\n", env)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func initialize() {
|
||||
config.TimeZone, _ = time.LoadLocation("America/Chicago")
|
||||
config.TimeZoneUTC, _ = time.LoadLocation("UTC")
|
||||
|
||||
// read command line options
|
||||
var (
|
||||
logLevel int
|
||||
ns1, ns2 string
|
||||
)
|
||||
|
||||
// log configuration
|
||||
flag.IntVar(&logLevel,
|
||||
"log",
|
||||
getEnvInt("LOG_LEVEL", 50),
|
||||
"(LOG_LEVEL)\nlog level")
|
||||
// http client configuration
|
||||
flag.IntVar(&config.HTTPClientRequestTimeout,
|
||||
"client-req-to",
|
||||
getEnvInt("HTTP_CLIENT_REQUEST_TIMEOUT", 60),
|
||||
"(HTTP_CLIENT_REQUEST_TIMEOUT)\ntime in seconds for the internal http client to complete a request")
|
||||
flag.IntVar(&config.HTTPClientConnectTimeout,
|
||||
"client-con-to",
|
||||
getEnvInt("HTTP_CLIENT_CONNECT_TIMEOUT", 5),
|
||||
"(HTTP_CLIENT_CONNECT_TIMEOUT)\ntime in seconds for the internal http client connection timeout")
|
||||
flag.IntVar(&config.HTTPClientTLSHandshakeTimeout,
|
||||
"client-tls-to",
|
||||
getEnvInt("HTTP_CLIENT_TLS_TIMEOUT", 5),
|
||||
"(HTTP_CLIENT_TLS_TIMEOUT)\ntime in seconds for the internal http client to complete a tls handshake")
|
||||
flag.IntVar(&config.HTTPClientIdleTimeout,
|
||||
"client-idle-to",
|
||||
getEnvInt("HTTP_CLIENT_IDLE_TIMEOUT", 5),
|
||||
"(HTTP_CLIENT_IDLE_TIMEOUT)\ntime in seconds that the internal http client will keep a connection open when idle")
|
||||
// Bind Config
|
||||
flag.StringVar(&config.NamedConfig.TTL,
|
||||
"bind-ttl",
|
||||
getEnvString("TTL", "1h"),
|
||||
"(TTL)\nBind zone time to live")
|
||||
flag.StringVar(&config.NamedConfig.Domain,
|
||||
"bind-domain",
|
||||
getEnvString("DOMAIN", "example.com"),
|
||||
"(DOMAIN)\nBind zone base domain")
|
||||
flag.StringVar(&config.NamedConfig.Email,
|
||||
"bind-email",
|
||||
getEnvString("EMAIL", "domain-admin@example.com"),
|
||||
"(EMAIL)\nBind zone authority e-mail address")
|
||||
flag.StringVar(&config.NamedConfig.Timestamp,
|
||||
"bind-timestamp",
|
||||
getEnvString("TIMESTAMP", time.Now().In(config.TimeZone).Format("0601021504")),
|
||||
"(TIMESTAMP)\nBind zone serial number")
|
||||
flag.StringVar(&config.NamedConfig.Refresh,
|
||||
"bind-refresh",
|
||||
getEnvString("REFRESH", "1h"),
|
||||
"(REFRESH)\nBind zone refresh time")
|
||||
flag.StringVar(&config.NamedConfig.Retry,
|
||||
"bind-retry",
|
||||
getEnvString("RETRY", "30m"),
|
||||
"(RETRY)\nBind zone retry time")
|
||||
flag.StringVar(&config.NamedConfig.Expire,
|
||||
"bind-expire",
|
||||
getEnvString("EXPIRE", "1w"),
|
||||
"(EXPIRE)\nBind zone expire time")
|
||||
flag.StringVar(&config.NamedConfig.Minimum,
|
||||
"bind-minimum",
|
||||
getEnvString("MINIMUM", "1h"),
|
||||
"(MINIMUM)\nBind zone minimum time")
|
||||
flag.StringVar(&ns1,
|
||||
"bind-ns1",
|
||||
getEnvString("NS1", ""),
|
||||
"(NS1)\nBind zone primary name-server")
|
||||
flag.StringVar(&ns2,
|
||||
"bind-ns2",
|
||||
getEnvString("NS2", ""),
|
||||
"(NS2)\nBind zone secondary name-server")
|
||||
// output file
|
||||
flag.StringVar(&config.BindOutputFileName,
|
||||
"filename",
|
||||
getEnvString("FILENAME", "./response-policy.bind"),
|
||||
"(FILENAME)\nWrite local file to filename")
|
||||
flag.Parse()
|
||||
|
||||
// set logging level
|
||||
switch {
|
||||
case logLevel <= 20:
|
||||
config.Log.SetMinLevel(logutils.LogLevel("ERROR"))
|
||||
case logLevel > 20 && logLevel <= 40:
|
||||
config.Log.SetMinLevel(logutils.LogLevel("WARNING"))
|
||||
case logLevel > 40 && logLevel <= 60:
|
||||
config.Log.SetMinLevel(logutils.LogLevel("INFO"))
|
||||
case logLevel > 60 && logLevel <= 80:
|
||||
config.Log.SetMinLevel(logutils.LogLevel("DEBUG"))
|
||||
case logLevel > 80:
|
||||
config.Log.SetMinLevel(logutils.LogLevel("TRACE"))
|
||||
}
|
||||
log.SetOutput(config.Log)
|
||||
|
||||
// print current configuration
|
||||
log.Printf("[DEBUG] configuration value set: LOG_LEVEL = %v\n", strconv.Itoa(logLevel))
|
||||
log.Printf("[DEBUG] configuration value set: HTTP_CLIENT_REQUEST_TIMEOUT = %v\n", strconv.Itoa(config.HTTPClientRequestTimeout))
|
||||
log.Printf("[DEBUG] configuration value set: HTTP_CLIENT_CONNECT_TIMEOUT = %v\n", strconv.Itoa(config.HTTPClientConnectTimeout))
|
||||
log.Printf("[DEBUG] configuration value set: HTTP_CLIENT_TLS_TIMEOUT = %v\n", strconv.Itoa(config.HTTPClientTLSHandshakeTimeout))
|
||||
log.Printf("[DEBUG] configuration value set: HTTP_CLIENT_IDLE_TIMEOUT = %v\n", strconv.Itoa(config.HTTPClientIdleTimeout))
|
||||
log.Printf("[DEBUG] configuration value set: TTL = %v\n", config.NamedConfig.TTL)
|
||||
log.Printf("[DEBUG] configuration value set: DOMAIN = %v\n", config.NamedConfig.Domain)
|
||||
log.Printf("[DEBUG] configuration value set: EMAIL = %v\n", config.NamedConfig.Email)
|
||||
log.Printf("[DEBUG] configuration value set: TIMESTAMP = %v\n", config.NamedConfig.Timestamp)
|
||||
log.Printf("[DEBUG] configuration value set: REFRESH = %v\n", config.NamedConfig.Refresh)
|
||||
log.Printf("[DEBUG] configuration value set: RETRY = %v\n", config.NamedConfig.Retry)
|
||||
log.Printf("[DEBUG] configuration value set: EXPIRE = %v\n", config.NamedConfig.Expire)
|
||||
log.Printf("[DEBUG] configuration value set: MINIMUM = %v\n", config.NamedConfig.Minimum)
|
||||
log.Printf("[DEBUG] configuration value set: NS1 = %v\n", ns1)
|
||||
log.Printf("[DEBUG] configuration value set: NS1 = %v\n", ns2)
|
||||
|
||||
// set bind-config nameservers
|
||||
if ns1 == "" {
|
||||
log.Printf("[ERROR] A primary name-server must be identified.")
|
||||
flag.PrintDefaults()
|
||||
os.Exit(1)
|
||||
} else {
|
||||
config.NamedConfig.NameServers = append(config.NamedConfig.NameServers, ns1)
|
||||
}
|
||||
if ns2 != "" {
|
||||
config.NamedConfig.NameServers = append(config.NamedConfig.NameServers, ns2)
|
||||
}
|
||||
config.NamedConfig.Email = strings.Replace(config.NamedConfig.Email, "@", ".", -1)
|
||||
|
||||
log.Printf("[DEBUG] Initialization Complete\n")
|
||||
}
|
||||
135
cmd/bind/main.go
Normal file
135
cmd/bind/main.go
Normal file
@ -0,0 +1,135 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"pihole-blocklist/v2/internal/httpclient"
|
||||
)
|
||||
|
||||
func main() {
|
||||
initialize()
|
||||
|
||||
// get remote URL data
|
||||
badDomains := getListData()
|
||||
|
||||
// clean-up
|
||||
config.NamedConfig.BadDomains = cleanBadDomains(badDomains)
|
||||
|
||||
buildBindResponsePolicyFile()
|
||||
}
|
||||
|
||||
func getListData() []string {
|
||||
var badDomains []string
|
||||
listSimple := make(chan []string)
|
||||
listComplex := make(chan []string)
|
||||
|
||||
// Get Simple Blocklists
|
||||
go func() {
|
||||
data := getData(config.URLBlocklistsSimple)
|
||||
domains := parseSimple(data)
|
||||
listSimple <- domains
|
||||
}()
|
||||
|
||||
// Get Host File Blocklists
|
||||
go func() {
|
||||
data := getData(config.URLBlocklistHostFiles)
|
||||
domains := parseComplex(data)
|
||||
listComplex <- domains
|
||||
}()
|
||||
|
||||
// Wait for all downloads to finish
|
||||
var (
|
||||
simple, complex []string
|
||||
simpleFinished, complexFinished bool
|
||||
)
|
||||
|
||||
for {
|
||||
select {
|
||||
case simple = <-listSimple:
|
||||
simpleFinished = true
|
||||
log.Printf("[INFO] All simple lists have been retrieved.\n")
|
||||
case complex = <-listComplex:
|
||||
log.Printf("[INFO] All complex lists have been retrieved.\n")
|
||||
complexFinished = true
|
||||
default:
|
||||
time.Sleep(time.Millisecond * 100)
|
||||
}
|
||||
|
||||
if simpleFinished && complexFinished {
|
||||
badDomains = append(badDomains, simple...)
|
||||
badDomains = append(badDomains, complex...)
|
||||
log.Printf("[INFO] Number of domains detected: %d\n", len(badDomains))
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return badDomains
|
||||
}
|
||||
|
||||
func getData(urls []string) []byte {
|
||||
log.Printf("[INFO] Downloading blocklists\n")
|
||||
var listData []byte
|
||||
|
||||
for _, u := range urls {
|
||||
log.Printf("[TRACE] Downloading URL: %s\n", u)
|
||||
c := httpclient.DefaultClient()
|
||||
data, err := c.Get(u)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Unable to get remote content from URL (%s): %v", u, err)
|
||||
}
|
||||
listData = append(listData, data...)
|
||||
}
|
||||
|
||||
return listData
|
||||
}
|
||||
|
||||
func cleanBadDomains(domains []string) []string {
|
||||
// remove duplicates
|
||||
total := len(domains)
|
||||
all := make(map[string]bool)
|
||||
list := []string{}
|
||||
for _, item := range domains {
|
||||
if _, value := all[item]; !value {
|
||||
all[item] = true
|
||||
list = append(list, item)
|
||||
}
|
||||
}
|
||||
domains = list
|
||||
log.Printf("[INFO] Duplicate items removed: %d\n", total-len(domains))
|
||||
|
||||
// remove hosts that are too long
|
||||
total = len(domains)
|
||||
list = []string{}
|
||||
for _, blocklistItem := range domains {
|
||||
if len([]rune(blocklistItem)) > 255 {
|
||||
continue
|
||||
}
|
||||
list = append(list, blocklistItem)
|
||||
}
|
||||
domains = list
|
||||
log.Printf("[INFO] Hosts with too many characters removed: %d\n", total-len(domains))
|
||||
|
||||
// remove allow-listed matches
|
||||
total = len(domains)
|
||||
list = []string{}
|
||||
for _, blocklistItem := range domains {
|
||||
var match bool
|
||||
for _, allowlistItem := range config.DomainAllowlist {
|
||||
if allowlistItem.MatchString(blocklistItem) {
|
||||
match = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !match {
|
||||
list = append(list, blocklistItem)
|
||||
}
|
||||
}
|
||||
domains = list
|
||||
log.Printf("[INFO] Allowed hosts removed: %d\n", total-len(domains))
|
||||
|
||||
log.Printf("[INFO] Total domains in list at end: %d.\n", len(domains))
|
||||
sort.Strings(domains)
|
||||
return domains
|
||||
}
|
||||
40
cmd/bind/parsing-complex.go
Normal file
40
cmd/bind/parsing-complex.go
Normal file
@ -0,0 +1,40 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/asaskevich/govalidator"
|
||||
)
|
||||
|
||||
func parseComplex(data []byte) []string {
|
||||
var domains []string
|
||||
|
||||
// convert data to reader for line-by-line reading
|
||||
r := bytes.NewReader(data)
|
||||
|
||||
// process combined files line-by-line
|
||||
scanner := bufio.NewScanner(r)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
|
||||
// skip lines where the first non-whitespace character is '#' or '//'
|
||||
if regexp.MustCompile(`^(\s+)?(#|\/\/)`).MatchString(line) {
|
||||
continue
|
||||
}
|
||||
|
||||
// split line by whitespace
|
||||
lineItems := strings.Fields(line)
|
||||
|
||||
if len(lineItems) >= 2 {
|
||||
// the second item is the domain, check if its valid and add it
|
||||
if govalidator.IsDNSName(lineItems[1]) {
|
||||
domains = append(domains, lineItems[1])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return domains
|
||||
}
|
||||
40
cmd/bind/parsing-simple.go
Normal file
40
cmd/bind/parsing-simple.go
Normal file
@ -0,0 +1,40 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/asaskevich/govalidator"
|
||||
)
|
||||
|
||||
func parseSimple(data []byte) []string {
|
||||
var domains []string
|
||||
|
||||
// convert data to reader for line-by-line reading
|
||||
r := bytes.NewReader(data)
|
||||
|
||||
// process combined files line-by-line
|
||||
scanner := bufio.NewScanner(r)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
|
||||
// skip lines where the first non-whitespace character is '#' or '//'
|
||||
if regexp.MustCompile(`^(\s+)?(#|\/\/)`).MatchString(line) {
|
||||
continue
|
||||
}
|
||||
|
||||
// split line by whitespace
|
||||
lineItems := strings.Fields(line)
|
||||
|
||||
if len(lineItems) >= 1 {
|
||||
// the second item is the domain, check if its valid and add it
|
||||
if govalidator.IsDNSName(lineItems[0]) {
|
||||
domains = append(domains, lineItems[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return domains
|
||||
}
|
||||
8
go.mod
Normal file
8
go.mod
Normal file
@ -0,0 +1,8 @@
|
||||
module pihole-blocklist/v2
|
||||
|
||||
go 1.17
|
||||
|
||||
require (
|
||||
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d
|
||||
github.com/hashicorp/logutils v1.0.0
|
||||
)
|
||||
4
go.sum
Normal file
4
go.sum
Normal file
@ -0,0 +1,4 @@
|
||||
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ=
|
||||
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
|
||||
github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y=
|
||||
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
|
||||
137
internal/httpclient/httpclient.go
Normal file
137
internal/httpclient/httpclient.go
Normal file
@ -0,0 +1,137 @@
|
||||
package httpclient
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"net"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// HTTPClient is an interface for initializing the http client library.
|
||||
type HTTPClient struct {
|
||||
Client *http.Client
|
||||
Data *bytes.Buffer
|
||||
Headers map[string]string
|
||||
|
||||
Username string
|
||||
Password string
|
||||
}
|
||||
|
||||
// DefaultClient is a function for defining a basic HTTP client with standard timeouts.
|
||||
func DefaultClient() *HTTPClient {
|
||||
return &HTTPClient{
|
||||
Client: &http.Client{
|
||||
Timeout: 60 * time.Second,
|
||||
Transport: &http.Transport{
|
||||
Dial: (&net.Dialer{
|
||||
Timeout: 5 * time.Second,
|
||||
}).Dial,
|
||||
TLSHandshakeTimeout: 5 * time.Second,
|
||||
IdleConnTimeout: 300 * time.Second,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// NewClient Create an HTTPClient with a user-provided net/http.Client
|
||||
func NewClient(httpClient *http.Client) *HTTPClient {
|
||||
return &HTTPClient{Client: httpClient}
|
||||
}
|
||||
|
||||
// SetBasicAuth is a chaining function to set the username and password for basic
|
||||
// authentication
|
||||
func (c *HTTPClient) SetBasicAuth(username, password string) *HTTPClient {
|
||||
c.Username = username
|
||||
c.Password = password
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
// SetPostData is a chaining function to set POST/PUT/PATCH data
|
||||
func (c *HTTPClient) SetPostData(data string) *HTTPClient {
|
||||
c.Data = bytes.NewBufferString(data)
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
// SetHeader is a chaining function to set arbitrary HTTP Headers
|
||||
func (c *HTTPClient) SetHeader(label string, value string) *HTTPClient {
|
||||
if c.Headers == nil {
|
||||
c.Headers = map[string]string{}
|
||||
}
|
||||
|
||||
c.Headers[label] = value
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
// Get calls the net.http GET operation
|
||||
func (c *HTTPClient) Get(url string) ([]byte, error) {
|
||||
return c.do(url, http.MethodGet)
|
||||
}
|
||||
|
||||
// Patch calls the net.http PATCH operation
|
||||
func (c *HTTPClient) Patch(url string) ([]byte, error) {
|
||||
return c.do(url, http.MethodPatch)
|
||||
}
|
||||
|
||||
// Post calls the net.http POST operation
|
||||
func (c *HTTPClient) Post(url string) ([]byte, error) {
|
||||
return c.do(url, http.MethodPost)
|
||||
}
|
||||
|
||||
// Put calls the net.http PUT operation
|
||||
func (c *HTTPClient) Put(url string) ([]byte, error) {
|
||||
return c.do(url, http.MethodPut)
|
||||
}
|
||||
|
||||
func (c *HTTPClient) do(url string, method string) ([]byte, error) {
|
||||
var (
|
||||
req *http.Request
|
||||
res *http.Response
|
||||
output []byte
|
||||
err error
|
||||
)
|
||||
|
||||
// NewRequest knows that c.data is typed *bytes.Buffer and will SEGFAULT
|
||||
// if c.data is nil. So we create a request using nil when c.data is nil
|
||||
if c.Data != nil {
|
||||
req, err = http.NewRequest(method, url, c.Data)
|
||||
} else {
|
||||
req, err = http.NewRequest(method, url, nil)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if (len(c.Username) > 0) && (len(c.Password) > 0) {
|
||||
req.SetBasicAuth(c.Username, c.Password)
|
||||
}
|
||||
|
||||
if c.Headers != nil {
|
||||
for label, value := range c.Headers {
|
||||
req.Header.Set(label, value)
|
||||
}
|
||||
}
|
||||
|
||||
if res, err = c.Client.Do(req); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer res.Body.Close()
|
||||
|
||||
if output, err = ioutil.ReadAll(res.Body); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// check status
|
||||
if res.StatusCode < 200 || res.StatusCode >= 300 {
|
||||
return nil, errors.New("non-successful status code received [" + strconv.Itoa(res.StatusCode) + "]")
|
||||
}
|
||||
|
||||
return output, nil
|
||||
}
|
||||
281
internal/httpclient/httpclient_test.go
Normal file
281
internal/httpclient/httpclient_test.go
Normal file
@ -0,0 +1,281 @@
|
||||
package httpclient
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
)
|
||||
|
||||
type Data struct {
|
||||
Greeting string `json:"greeting"`
|
||||
Headers map[string]string `json:"headers"`
|
||||
Method string `json:"method"`
|
||||
Username string `json:"username"`
|
||||
Password string `json:"password"`
|
||||
PostData string `json:"postdata"`
|
||||
}
|
||||
|
||||
var (
|
||||
greeting = "Hello world"
|
||||
postData = "Test data"
|
||||
authUser = "testuser"
|
||||
authPass = "testpass"
|
||||
headerLabel = "Test-Header"
|
||||
headerValue = "Test-Value"
|
||||
)
|
||||
|
||||
func httpTestHandler(w http.ResponseWriter, r *http.Request) {
|
||||
var (
|
||||
b []byte
|
||||
user string
|
||||
pass string
|
||||
body []byte
|
||||
)
|
||||
|
||||
data := Data{
|
||||
Greeting: greeting,
|
||||
Headers: map[string]string{},
|
||||
Method: r.Method,
|
||||
}
|
||||
|
||||
user, pass, ok := r.BasicAuth()
|
||||
if ok {
|
||||
data.Username = user
|
||||
data.Password = pass
|
||||
}
|
||||
|
||||
body, err := ioutil.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
fmt.Fprint(w, "ioutil.ReadAll failed")
|
||||
}
|
||||
data.PostData = string(body)
|
||||
|
||||
for h := range r.Header {
|
||||
data.Headers[h] = r.Header.Get(h)
|
||||
}
|
||||
|
||||
b, err = json.MarshalIndent(data, "", " ")
|
||||
if err != nil {
|
||||
fmt.Fprint(w, "Json marshal failed somehow")
|
||||
}
|
||||
fmt.Fprint(w, string(b))
|
||||
}
|
||||
|
||||
func checkMethod(t *testing.T, data Data, method string) {
|
||||
if data.Method != method {
|
||||
t.Errorf("data.Method(%s) != method(%s)", data.Method, method)
|
||||
}
|
||||
t.Log("checkMethod() success")
|
||||
}
|
||||
|
||||
func checkGreeting(t *testing.T, data Data) {
|
||||
if data.Greeting != greeting {
|
||||
t.Errorf("data.Greeting(%s) != greeting(%s)", data.Greeting, greeting)
|
||||
}
|
||||
t.Log("checkGreeting() success")
|
||||
}
|
||||
|
||||
func checkBasicAuth(t *testing.T, data Data) {
|
||||
if data.Username != authUser {
|
||||
t.Errorf("data.Username(%s) != authUser(%s)", data.Username, authUser)
|
||||
}
|
||||
if data.Password != authPass {
|
||||
t.Errorf("data.Password(%s) != authPass(%s)", data.Password, authPass)
|
||||
}
|
||||
t.Log("checkBasicAuth() success")
|
||||
}
|
||||
|
||||
func checkPostData(t *testing.T, data Data) {
|
||||
if data.PostData != postData {
|
||||
t.Errorf("data.PostData(%s) != postData(%s)", data.PostData, postData)
|
||||
}
|
||||
t.Log("checkPostData() success")
|
||||
}
|
||||
|
||||
func TestGet(t *testing.T) {
|
||||
var data Data
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(httpTestHandler))
|
||||
defer ts.Close()
|
||||
|
||||
output, err := DefaultClient().Get(ts.URL)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if err = json.Unmarshal(output, &data); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
checkMethod(t, data, http.MethodGet)
|
||||
checkGreeting(t, data)
|
||||
}
|
||||
|
||||
func TestGetAuth(t *testing.T) {
|
||||
var data Data
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(httpTestHandler))
|
||||
defer ts.Close()
|
||||
|
||||
output, err := DefaultClient().SetBasicAuth(authUser, authPass).Get(ts.URL)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if err = json.Unmarshal(output, &data); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
checkMethod(t, data, http.MethodGet)
|
||||
checkGreeting(t, data)
|
||||
checkBasicAuth(t, data)
|
||||
}
|
||||
|
||||
func TestPut(t *testing.T) {
|
||||
var data Data
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(httpTestHandler))
|
||||
defer ts.Close()
|
||||
|
||||
output, err := DefaultClient().SetPostData(postData).Put(ts.URL)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if err = json.Unmarshal(output, &data); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
checkMethod(t, data, http.MethodPut)
|
||||
checkGreeting(t, data)
|
||||
checkPostData(t, data)
|
||||
}
|
||||
|
||||
func TestPutAuth(t *testing.T) {
|
||||
var data Data
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(httpTestHandler))
|
||||
defer ts.Close()
|
||||
|
||||
output, err := DefaultClient().SetBasicAuth(authUser, authPass).SetPostData(postData).Put(ts.URL)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if err = json.Unmarshal(output, &data); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
checkMethod(t, data, http.MethodPut)
|
||||
checkGreeting(t, data)
|
||||
checkBasicAuth(t, data)
|
||||
checkPostData(t, data)
|
||||
}
|
||||
|
||||
func TestPost(t *testing.T) {
|
||||
var data Data
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(httpTestHandler))
|
||||
defer ts.Close()
|
||||
|
||||
output, err := DefaultClient().SetPostData(postData).Post(ts.URL)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if err = json.Unmarshal(output, &data); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
checkMethod(t, data, http.MethodPost)
|
||||
checkGreeting(t, data)
|
||||
checkPostData(t, data)
|
||||
}
|
||||
|
||||
func TestPostAuth(t *testing.T) {
|
||||
var data Data
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(httpTestHandler))
|
||||
defer ts.Close()
|
||||
|
||||
output, err := DefaultClient().SetBasicAuth(authUser, authPass).SetPostData(postData).Post(ts.URL)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if err = json.Unmarshal(output, &data); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
checkMethod(t, data, http.MethodPost)
|
||||
checkGreeting(t, data)
|
||||
checkBasicAuth(t, data)
|
||||
checkPostData(t, data)
|
||||
}
|
||||
|
||||
func TestPatch(t *testing.T) {
|
||||
var data Data
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(httpTestHandler))
|
||||
defer ts.Close()
|
||||
|
||||
output, err := DefaultClient().SetPostData(postData).Patch(ts.URL)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if err = json.Unmarshal(output, &data); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
checkMethod(t, data, http.MethodPatch)
|
||||
checkGreeting(t, data)
|
||||
checkPostData(t, data)
|
||||
}
|
||||
|
||||
func TestPatchAuth(t *testing.T) {
|
||||
var data Data
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(httpTestHandler))
|
||||
defer ts.Close()
|
||||
|
||||
output, err := DefaultClient().SetBasicAuth(authUser, authPass).SetPostData(postData).Patch(ts.URL)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if err = json.Unmarshal(output, &data); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
checkMethod(t, data, http.MethodPatch)
|
||||
checkGreeting(t, data)
|
||||
checkBasicAuth(t, data)
|
||||
checkPostData(t, data)
|
||||
}
|
||||
|
||||
func TestSetHeader(t *testing.T) {
|
||||
var data Data
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(httpTestHandler))
|
||||
defer ts.Close()
|
||||
|
||||
output, err := DefaultClient().SetHeader(headerLabel, headerValue).Get(ts.URL)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if err = json.Unmarshal(output, &data); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
checkMethod(t, data, http.MethodGet)
|
||||
checkGreeting(t, data)
|
||||
if data.Headers[headerLabel] != headerValue {
|
||||
t.Errorf("SetHeader values not set in header: %+v", data.Headers)
|
||||
}
|
||||
}
|
||||
23
templates/bind-response-policy.template
Normal file
23
templates/bind-response-policy.template
Normal file
@ -0,0 +1,23 @@
|
||||
{{- $domain := .Domain -}}
|
||||
$TTL {{ or .TTL "1h" }}
|
||||
@ IN SOA {{ $domain }}. {{ or .Email "domain-admin" }}. (
|
||||
{{ or .Timestamp "0000000000" }} ; Serial
|
||||
{{ or .Refresh "1h" }} ; Refresh
|
||||
{{ or .Retry "30m" }} ; Retry
|
||||
{{ or .Expire "1w" }} ; Expire
|
||||
{{ or .Minimum "1h" }} ; Minimum
|
||||
)
|
||||
|
||||
;
|
||||
; Name Servers
|
||||
;
|
||||
{{- range .NameServers }}
|
||||
IN NS {{ . }}.
|
||||
{{- end }}
|
||||
|
||||
;
|
||||
; Addresses
|
||||
;
|
||||
{{- range .BadDomains }}
|
||||
{{ . }} IN CNAME blocked.{{ $domain }}.
|
||||
{{- end }}
|
||||
Loading…
x
Reference in New Issue
Block a user