initial commit
This commit is contained in:
61
cmd/bind/build-bind.go
Normal file
61
cmd/bind/build-bind.go
Normal file
@@ -0,0 +1,61 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"text/template"
|
||||
)
|
||||
|
||||
func buildBindResponsePolicyFile() {
|
||||
var (
|
||||
output bytes.Buffer
|
||||
)
|
||||
|
||||
outputTemplate := `{{- $domain := .Domain -}}
|
||||
$TTL {{ or .TTL "1h" }}
|
||||
@ IN SOA {{ $domain }}. {{ or .Email "domain-admin" }}. (
|
||||
{{ or .Timestamp "0000000000" }} ; Serial
|
||||
{{ or .Refresh "1h" }} ; Refresh
|
||||
{{ or .Retry "30m" }} ; Retry
|
||||
{{ or .Expire "1w" }} ; Expire
|
||||
{{ or .Minimum "1h" }} ; Minimum
|
||||
)
|
||||
|
||||
;
|
||||
; Name Servers
|
||||
;
|
||||
{{- range .NameServers }}
|
||||
IN NS {{ . }}.
|
||||
{{- end }}
|
||||
|
||||
;
|
||||
; Addresses
|
||||
;
|
||||
{{- range .BadDomains }}
|
||||
{{ . }} IN CNAME blocked.{{ $domain }}.
|
||||
{{- end }}`
|
||||
|
||||
t, err := template.New("response-policy-zone").Parse(outputTemplate)
|
||||
if err != nil {
|
||||
log.Fatalf("[FATAL] Unable to parse template (%s): %v\n", "response-policy-zone", err)
|
||||
}
|
||||
|
||||
if err := t.Execute(&output, config.NamedConfig); err != nil {
|
||||
log.Fatalf("[FATAL] Unable to generate template output: %v\n", err)
|
||||
}
|
||||
|
||||
fileWriter, err := os.Create(config.BindOutputFileName)
|
||||
if err != nil {
|
||||
log.Fatalf("[FATAL] Unable to open file (%s) for writing: %v", config.BindOutputFileName, err)
|
||||
}
|
||||
defer fileWriter.Close()
|
||||
|
||||
bytesWritten, err := fileWriter.Write(output.Bytes())
|
||||
if err != nil {
|
||||
log.Fatalf("[FATAL] Unable to write to file (%s): %v", config.BindOutputFileName, err)
|
||||
}
|
||||
|
||||
log.Printf("[DEBUG] Wrote %d bytes to %s.\n", bytesWritten, config.BindOutputFileName)
|
||||
}
|
||||
118
cmd/bind/config.go
Normal file
118
cmd/bind/config.go
Normal file
@@ -0,0 +1,118 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/logutils"
|
||||
)
|
||||
|
||||
type configStructure struct {
|
||||
// time configuration
|
||||
TimeFormat string
|
||||
TimeZone *time.Location
|
||||
TimeZoneUTC *time.Location
|
||||
|
||||
// logging
|
||||
Log *logutils.LevelFilter
|
||||
|
||||
// HTTP Client timeout configurations
|
||||
HTTPClientRequestTimeout int
|
||||
HTTPClientConnectTimeout int
|
||||
HTTPClientTLSHandshakeTimeout int
|
||||
HTTPClientIdleTimeout int
|
||||
|
||||
// Download Sources
|
||||
URLBlocklistHostFiles []string
|
||||
URLBlocklistsSimple []string
|
||||
|
||||
// Allowlist (regex)
|
||||
DomainAllowlist []*regexp.Regexp
|
||||
|
||||
// Named Config Generator
|
||||
NamedConfig namedConfigStruct
|
||||
|
||||
// Output Filename
|
||||
BindOutputFileName string
|
||||
}
|
||||
|
||||
type namedConfigStruct struct {
|
||||
TTL string
|
||||
Domain string
|
||||
Email string
|
||||
Timestamp string
|
||||
Refresh string
|
||||
Retry string
|
||||
Expire string
|
||||
Minimum string
|
||||
NameServers []string
|
||||
BadDomains []string
|
||||
}
|
||||
|
||||
var config = configStructure{
|
||||
TimeFormat: "2006-01-02 15:04:05",
|
||||
Log: &logutils.LevelFilter{
|
||||
Levels: []logutils.LogLevel{"TRACE", "DEBUG", "INFO", "WARNING", "ERROR"},
|
||||
Writer: os.Stderr,
|
||||
},
|
||||
|
||||
// Nice blocklist location: https://firebog.net/
|
||||
// Default Blocklist
|
||||
URLBlocklistHostFiles: []string{
|
||||
"https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts",
|
||||
"http://sysctl.org/cameleon/hosts",
|
||||
"https://raw.githubusercontent.com/DandelionSprout/adfilt/master/Alternate%20versions%20Anti-Malware%20List/AntiMalwareHosts.txt",
|
||||
"https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Risk/hosts",
|
||||
},
|
||||
URLBlocklistsSimple: []string{
|
||||
"https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt",
|
||||
"https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt",
|
||||
"https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt",
|
||||
"https://v.firebog.net/hosts/Prigent-Crypto.txt",
|
||||
"https://phishing.army/download/phishing_army_blocklist_extended.txt",
|
||||
"https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-malware.txt",
|
||||
"https://raw.githubusercontent.com/Spam404/lists/master/main-blacklist.txt",
|
||||
"https://dbl.oisd.nl/",
|
||||
"https://osint.digitalside.it/Threat-Intel/lists/latestdomains.txt",
|
||||
|
||||
},
|
||||
|
||||
// default URL Allow hosts
|
||||
DomainAllowlist: []*regexp.Regexp{
|
||||
// localhosts included in blocklists for some reason
|
||||
regexp.MustCompile(`localhost`),
|
||||
regexp.MustCompile(`localhost.localdomain`),
|
||||
regexp.MustCompile(`local`),
|
||||
regexp.MustCompile(`broadcasthost`),
|
||||
regexp.MustCompile(`localhost`),
|
||||
regexp.MustCompile(`ip6-localhost`),
|
||||
regexp.MustCompile(`ip6-loopback`),
|
||||
regexp.MustCompile(`localhost`),
|
||||
regexp.MustCompile(`ip6-localnet`),
|
||||
regexp.MustCompile(`ip6-mcastprefix`),
|
||||
regexp.MustCompile(`ip6-allnodes`),
|
||||
regexp.MustCompile(`ip6-allrouters`),
|
||||
regexp.MustCompile(`ip6-allhosts`),
|
||||
// default allow hosts
|
||||
regexp.MustCompile(`(^|\.)` + `thepiratebay\.org`),
|
||||
regexp.MustCompile(`(^|\.)` + `sendgrid\.net`),
|
||||
regexp.MustCompile(`(^|\.)` + `googleadservices\.com`),
|
||||
regexp.MustCompile(`(^|\.)` + `doubleclick\.net`),
|
||||
regexp.MustCompile(`(^|\.)` + `sailthru\.com`),
|
||||
regexp.MustCompile(`(^|\.)` + `magiskmanager\.com`),
|
||||
regexp.MustCompile(`(^|\.)` + `apiservices\.krxd\.net`),
|
||||
regexp.MustCompile(`(^|\.)` + `logfiles\.zoom\.us`),
|
||||
regexp.MustCompile(`(^|\.)` + `logfiles-va\.zoom\.us`),
|
||||
regexp.MustCompile(`(^|\.)` + `nest\.com`),
|
||||
regexp.MustCompile(`(^|\.)` + `clients.\.google\.com`),
|
||||
regexp.MustCompile(`(^|\.)` + `login\.live\.com`),
|
||||
regexp.MustCompile(`(^|\.)` + `unagi\.amazon\.com`),
|
||||
regexp.MustCompile(`(^|\.)` + `unagi-na\.amazon\.com`),
|
||||
regexp.MustCompile(`(^|\.)` + `duckduckgo\.com`),
|
||||
regexp.MustCompile(`(^|\.)` + `msn\.com`),
|
||||
regexp.MustCompile(`(^|\.)` + `nexusrules\.officeapps\.live\.com`),
|
||||
regexp.MustCompile(`(^|\.)` + `playfabapi\.com`),
|
||||
regexp.MustCompile(`(^|\.)` + `vercel-dns\.com`),
|
||||
},
|
||||
}
|
||||
167
cmd/bind/init.go
Normal file
167
cmd/bind/init.go
Normal file
@@ -0,0 +1,167 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/logutils"
|
||||
)
|
||||
|
||||
// getEnvString returns string from environment variable
|
||||
func getEnvString(env, def string) (val string) { //nolint:deadcode
|
||||
val = os.Getenv(env)
|
||||
|
||||
if val == "" {
|
||||
return def
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// getEnvInt returns int from environment variable
|
||||
func getEnvInt(env string, def int) (ret int) {
|
||||
val := os.Getenv(env)
|
||||
|
||||
if val == "" {
|
||||
return def
|
||||
}
|
||||
|
||||
ret, err := strconv.Atoi(val)
|
||||
if err != nil {
|
||||
log.Fatalf("[ERROR] Environment variable is not numeric: %v\n", env)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func initialize() {
|
||||
config.TimeZone, _ = time.LoadLocation("America/Chicago")
|
||||
config.TimeZoneUTC, _ = time.LoadLocation("UTC")
|
||||
|
||||
// read command line options
|
||||
var (
|
||||
logLevel int
|
||||
ns1, ns2 string
|
||||
)
|
||||
|
||||
// log configuration
|
||||
flag.IntVar(&logLevel,
|
||||
"log",
|
||||
getEnvInt("LOG_LEVEL", 50),
|
||||
"(LOG_LEVEL)\nlog level")
|
||||
// http client configuration
|
||||
flag.IntVar(&config.HTTPClientRequestTimeout,
|
||||
"client-req-to",
|
||||
getEnvInt("HTTP_CLIENT_REQUEST_TIMEOUT", 60),
|
||||
"(HTTP_CLIENT_REQUEST_TIMEOUT)\ntime in seconds for the internal http client to complete a request")
|
||||
flag.IntVar(&config.HTTPClientConnectTimeout,
|
||||
"client-con-to",
|
||||
getEnvInt("HTTP_CLIENT_CONNECT_TIMEOUT", 5),
|
||||
"(HTTP_CLIENT_CONNECT_TIMEOUT)\ntime in seconds for the internal http client connection timeout")
|
||||
flag.IntVar(&config.HTTPClientTLSHandshakeTimeout,
|
||||
"client-tls-to",
|
||||
getEnvInt("HTTP_CLIENT_TLS_TIMEOUT", 5),
|
||||
"(HTTP_CLIENT_TLS_TIMEOUT)\ntime in seconds for the internal http client to complete a tls handshake")
|
||||
flag.IntVar(&config.HTTPClientIdleTimeout,
|
||||
"client-idle-to",
|
||||
getEnvInt("HTTP_CLIENT_IDLE_TIMEOUT", 5),
|
||||
"(HTTP_CLIENT_IDLE_TIMEOUT)\ntime in seconds that the internal http client will keep a connection open when idle")
|
||||
// Bind Config
|
||||
flag.StringVar(&config.NamedConfig.TTL,
|
||||
"bind-ttl",
|
||||
getEnvString("TTL", "1h"),
|
||||
"(TTL)\nBind zone time to live")
|
||||
flag.StringVar(&config.NamedConfig.Domain,
|
||||
"bind-domain",
|
||||
getEnvString("DOMAIN", "example.com"),
|
||||
"(DOMAIN)\nBind zone base domain")
|
||||
flag.StringVar(&config.NamedConfig.Email,
|
||||
"bind-email",
|
||||
getEnvString("EMAIL", "domain-admin@example.com"),
|
||||
"(EMAIL)\nBind zone authority e-mail address")
|
||||
flag.StringVar(&config.NamedConfig.Timestamp,
|
||||
"bind-timestamp",
|
||||
getEnvString("TIMESTAMP", time.Now().In(config.TimeZone).Format("0601021504")),
|
||||
"(TIMESTAMP)\nBind zone serial number")
|
||||
flag.StringVar(&config.NamedConfig.Refresh,
|
||||
"bind-refresh",
|
||||
getEnvString("REFRESH", "1h"),
|
||||
"(REFRESH)\nBind zone refresh time")
|
||||
flag.StringVar(&config.NamedConfig.Retry,
|
||||
"bind-retry",
|
||||
getEnvString("RETRY", "30m"),
|
||||
"(RETRY)\nBind zone retry time")
|
||||
flag.StringVar(&config.NamedConfig.Expire,
|
||||
"bind-expire",
|
||||
getEnvString("EXPIRE", "1w"),
|
||||
"(EXPIRE)\nBind zone expire time")
|
||||
flag.StringVar(&config.NamedConfig.Minimum,
|
||||
"bind-minimum",
|
||||
getEnvString("MINIMUM", "1h"),
|
||||
"(MINIMUM)\nBind zone minimum time")
|
||||
flag.StringVar(&ns1,
|
||||
"bind-ns1",
|
||||
getEnvString("NS1", ""),
|
||||
"(NS1)\nBind zone primary name-server")
|
||||
flag.StringVar(&ns2,
|
||||
"bind-ns2",
|
||||
getEnvString("NS2", ""),
|
||||
"(NS2)\nBind zone secondary name-server")
|
||||
// output file
|
||||
flag.StringVar(&config.BindOutputFileName,
|
||||
"filename",
|
||||
getEnvString("FILENAME", "./response-policy.bind"),
|
||||
"(FILENAME)\nWrite local file to filename")
|
||||
flag.Parse()
|
||||
|
||||
// set logging level
|
||||
switch {
|
||||
case logLevel <= 20:
|
||||
config.Log.SetMinLevel(logutils.LogLevel("ERROR"))
|
||||
case logLevel > 20 && logLevel <= 40:
|
||||
config.Log.SetMinLevel(logutils.LogLevel("WARNING"))
|
||||
case logLevel > 40 && logLevel <= 60:
|
||||
config.Log.SetMinLevel(logutils.LogLevel("INFO"))
|
||||
case logLevel > 60 && logLevel <= 80:
|
||||
config.Log.SetMinLevel(logutils.LogLevel("DEBUG"))
|
||||
case logLevel > 80:
|
||||
config.Log.SetMinLevel(logutils.LogLevel("TRACE"))
|
||||
}
|
||||
log.SetOutput(config.Log)
|
||||
|
||||
// print current configuration
|
||||
log.Printf("[DEBUG] configuration value set: LOG_LEVEL = %v\n", strconv.Itoa(logLevel))
|
||||
log.Printf("[DEBUG] configuration value set: HTTP_CLIENT_REQUEST_TIMEOUT = %v\n", strconv.Itoa(config.HTTPClientRequestTimeout))
|
||||
log.Printf("[DEBUG] configuration value set: HTTP_CLIENT_CONNECT_TIMEOUT = %v\n", strconv.Itoa(config.HTTPClientConnectTimeout))
|
||||
log.Printf("[DEBUG] configuration value set: HTTP_CLIENT_TLS_TIMEOUT = %v\n", strconv.Itoa(config.HTTPClientTLSHandshakeTimeout))
|
||||
log.Printf("[DEBUG] configuration value set: HTTP_CLIENT_IDLE_TIMEOUT = %v\n", strconv.Itoa(config.HTTPClientIdleTimeout))
|
||||
log.Printf("[DEBUG] configuration value set: TTL = %v\n", config.NamedConfig.TTL)
|
||||
log.Printf("[DEBUG] configuration value set: DOMAIN = %v\n", config.NamedConfig.Domain)
|
||||
log.Printf("[DEBUG] configuration value set: EMAIL = %v\n", config.NamedConfig.Email)
|
||||
log.Printf("[DEBUG] configuration value set: TIMESTAMP = %v\n", config.NamedConfig.Timestamp)
|
||||
log.Printf("[DEBUG] configuration value set: REFRESH = %v\n", config.NamedConfig.Refresh)
|
||||
log.Printf("[DEBUG] configuration value set: RETRY = %v\n", config.NamedConfig.Retry)
|
||||
log.Printf("[DEBUG] configuration value set: EXPIRE = %v\n", config.NamedConfig.Expire)
|
||||
log.Printf("[DEBUG] configuration value set: MINIMUM = %v\n", config.NamedConfig.Minimum)
|
||||
log.Printf("[DEBUG] configuration value set: NS1 = %v\n", ns1)
|
||||
log.Printf("[DEBUG] configuration value set: NS1 = %v\n", ns2)
|
||||
|
||||
// set bind-config nameservers
|
||||
if ns1 == "" {
|
||||
log.Printf("[ERROR] A primary name-server must be identified.")
|
||||
flag.PrintDefaults()
|
||||
os.Exit(1)
|
||||
} else {
|
||||
config.NamedConfig.NameServers = append(config.NamedConfig.NameServers, ns1)
|
||||
}
|
||||
if ns2 != "" {
|
||||
config.NamedConfig.NameServers = append(config.NamedConfig.NameServers, ns2)
|
||||
}
|
||||
config.NamedConfig.Email = strings.Replace(config.NamedConfig.Email, "@", ".", -1)
|
||||
|
||||
log.Printf("[DEBUG] Initialization Complete\n")
|
||||
}
|
||||
135
cmd/bind/main.go
Normal file
135
cmd/bind/main.go
Normal file
@@ -0,0 +1,135 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"pihole-blocklist/v2/internal/httpclient"
|
||||
)
|
||||
|
||||
func main() {
|
||||
initialize()
|
||||
|
||||
// get remote URL data
|
||||
badDomains := getListData()
|
||||
|
||||
// clean-up
|
||||
config.NamedConfig.BadDomains = cleanBadDomains(badDomains)
|
||||
|
||||
buildBindResponsePolicyFile()
|
||||
}
|
||||
|
||||
func getListData() []string {
|
||||
var badDomains []string
|
||||
listSimple := make(chan []string)
|
||||
listComplex := make(chan []string)
|
||||
|
||||
// Get Simple Blocklists
|
||||
go func() {
|
||||
data := getData(config.URLBlocklistsSimple)
|
||||
domains := parseSimple(data)
|
||||
listSimple <- domains
|
||||
}()
|
||||
|
||||
// Get Host File Blocklists
|
||||
go func() {
|
||||
data := getData(config.URLBlocklistHostFiles)
|
||||
domains := parseComplex(data)
|
||||
listComplex <- domains
|
||||
}()
|
||||
|
||||
// Wait for all downloads to finish
|
||||
var (
|
||||
simple, complex []string
|
||||
simpleFinished, complexFinished bool
|
||||
)
|
||||
|
||||
for {
|
||||
select {
|
||||
case simple = <-listSimple:
|
||||
simpleFinished = true
|
||||
log.Printf("[INFO] All simple lists have been retrieved.\n")
|
||||
case complex = <-listComplex:
|
||||
log.Printf("[INFO] All complex lists have been retrieved.\n")
|
||||
complexFinished = true
|
||||
default:
|
||||
time.Sleep(time.Millisecond * 100)
|
||||
}
|
||||
|
||||
if simpleFinished && complexFinished {
|
||||
badDomains = append(badDomains, simple...)
|
||||
badDomains = append(badDomains, complex...)
|
||||
log.Printf("[INFO] Number of domains detected: %d\n", len(badDomains))
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return badDomains
|
||||
}
|
||||
|
||||
func getData(urls []string) []byte {
|
||||
log.Printf("[INFO] Downloading blocklists\n")
|
||||
var listData []byte
|
||||
|
||||
for _, u := range urls {
|
||||
log.Printf("[TRACE] Downloading URL: %s\n", u)
|
||||
c := httpclient.DefaultClient()
|
||||
data, err := c.Get(u)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Unable to get remote content from URL (%s): %v", u, err)
|
||||
}
|
||||
listData = append(listData, data...)
|
||||
}
|
||||
|
||||
return listData
|
||||
}
|
||||
|
||||
func cleanBadDomains(domains []string) []string {
|
||||
// remove duplicates
|
||||
total := len(domains)
|
||||
all := make(map[string]bool)
|
||||
list := []string{}
|
||||
for _, item := range domains {
|
||||
if _, value := all[item]; !value {
|
||||
all[item] = true
|
||||
list = append(list, item)
|
||||
}
|
||||
}
|
||||
domains = list
|
||||
log.Printf("[INFO] Duplicate items removed: %d\n", total-len(domains))
|
||||
|
||||
// remove hosts that are too long
|
||||
total = len(domains)
|
||||
list = []string{}
|
||||
for _, blocklistItem := range domains {
|
||||
if len([]rune(blocklistItem)) > 255 {
|
||||
continue
|
||||
}
|
||||
list = append(list, blocklistItem)
|
||||
}
|
||||
domains = list
|
||||
log.Printf("[INFO] Hosts with too many characters removed: %d\n", total-len(domains))
|
||||
|
||||
// remove allow-listed matches
|
||||
total = len(domains)
|
||||
list = []string{}
|
||||
for _, blocklistItem := range domains {
|
||||
var match bool
|
||||
for _, allowlistItem := range config.DomainAllowlist {
|
||||
if allowlistItem.MatchString(blocklistItem) {
|
||||
match = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !match {
|
||||
list = append(list, blocklistItem)
|
||||
}
|
||||
}
|
||||
domains = list
|
||||
log.Printf("[INFO] Allowed hosts removed: %d\n", total-len(domains))
|
||||
|
||||
log.Printf("[INFO] Total domains in list at end: %d.\n", len(domains))
|
||||
sort.Strings(domains)
|
||||
return domains
|
||||
}
|
||||
40
cmd/bind/parsing-complex.go
Normal file
40
cmd/bind/parsing-complex.go
Normal file
@@ -0,0 +1,40 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/asaskevich/govalidator"
|
||||
)
|
||||
|
||||
func parseComplex(data []byte) []string {
|
||||
var domains []string
|
||||
|
||||
// convert data to reader for line-by-line reading
|
||||
r := bytes.NewReader(data)
|
||||
|
||||
// process combined files line-by-line
|
||||
scanner := bufio.NewScanner(r)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
|
||||
// skip lines where the first non-whitespace character is '#' or '//'
|
||||
if regexp.MustCompile(`^(\s+)?(#|\/\/)`).MatchString(line) {
|
||||
continue
|
||||
}
|
||||
|
||||
// split line by whitespace
|
||||
lineItems := strings.Fields(line)
|
||||
|
||||
if len(lineItems) >= 2 {
|
||||
// the second item is the domain, check if its valid and add it
|
||||
if govalidator.IsDNSName(lineItems[1]) {
|
||||
domains = append(domains, lineItems[1])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return domains
|
||||
}
|
||||
40
cmd/bind/parsing-simple.go
Normal file
40
cmd/bind/parsing-simple.go
Normal file
@@ -0,0 +1,40 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/asaskevich/govalidator"
|
||||
)
|
||||
|
||||
func parseSimple(data []byte) []string {
|
||||
var domains []string
|
||||
|
||||
// convert data to reader for line-by-line reading
|
||||
r := bytes.NewReader(data)
|
||||
|
||||
// process combined files line-by-line
|
||||
scanner := bufio.NewScanner(r)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
|
||||
// skip lines where the first non-whitespace character is '#' or '//'
|
||||
if regexp.MustCompile(`^(\s+)?(#|\/\/)`).MatchString(line) {
|
||||
continue
|
||||
}
|
||||
|
||||
// split line by whitespace
|
||||
lineItems := strings.Fields(line)
|
||||
|
||||
if len(lineItems) >= 1 {
|
||||
// the second item is the domain, check if its valid and add it
|
||||
if govalidator.IsDNSName(lineItems[0]) {
|
||||
domains = append(domains, lineItems[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return domains
|
||||
}
|
||||
Reference in New Issue
Block a user