65 lines
1.2 KiB
Go
65 lines
1.2 KiB
Go
package main
|
|
|
|
import (
|
|
"regexp"
|
|
"sort"
|
|
|
|
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/log"
|
|
)
|
|
|
|
func removeDuplicateStr(domains []string) []string {
|
|
allKeys := make(map[string]bool)
|
|
allDomains := []string{}
|
|
for _, item := range domains {
|
|
if _, value := allKeys[item]; !value {
|
|
allKeys[item] = true
|
|
allDomains = append(allDomains, item)
|
|
}
|
|
}
|
|
return allDomains
|
|
}
|
|
|
|
func cleanBadDomains(domains []string) []string {
|
|
var (
|
|
cleanDomains []string
|
|
t int64
|
|
)
|
|
|
|
// remove duplicates
|
|
domains = removeDuplicateStr(domains)
|
|
|
|
for _, domain := range domains {
|
|
// removing trailing dots
|
|
domain = regexp.MustCompile(`\.$`).ReplaceAllString(domain, "")
|
|
|
|
// skip domains that are too long
|
|
if len([]rune(domain)) > 230 {
|
|
continue
|
|
}
|
|
|
|
// skip domains that are allowed
|
|
var skip bool
|
|
for _, allowRegex := range cfg.ConfigFile.AllowLists {
|
|
if regexp.MustCompile(allowRegex).MatchString(domain) {
|
|
skip = true
|
|
continue
|
|
}
|
|
}
|
|
if skip {
|
|
log.Trace("Skipping Domain", "domain", domain)
|
|
continue
|
|
}
|
|
|
|
// add domain
|
|
cleanDomains = append(cleanDomains, domain)
|
|
|
|
if t%10000 == 0 {
|
|
log.Debug("Clean-Up", "number", t)
|
|
}
|
|
t++
|
|
}
|
|
|
|
sort.Strings(cleanDomains)
|
|
return cleanDomains
|
|
}
|