improve performace?

cmd/bind/cleanup.go

@@ -7,63 +7,52 @@ import (
 	"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/log"
 )
 
-func cleanBadDomains(domains []string) []string {
+func removeDuplicateStr(domains []string) []string {
-	// remove duplicates
+	allKeys := make(map[string]bool)
-	total := len(domains)
+	allDomains := []string{}
-	all := make(map[string]bool)
-	list := []string{}
 	for _, item := range domains {
-		if _, value := all[item]; !value {
+		if _, value := allKeys[item]; !value {
-			all[item] = true
+			allKeys[item] = true
-			list = append(list, item)
+			allDomains = append(allDomains, item)
 		}
 	}
-	domains = list
+	return allDomains
-	log.Info("hosts removed from blocklist", "reason", "duplicate", "hosts", total-len(domains))
+}
 
-	// remove hosts that are too long
+func cleanBadDomains(domains []string) []string {
-	total = len(domains)
+	var (
-	list = []string{}
+		cleanDomains []string
-	for _, blocklistItem := range domains {
+		t            int64
-		if len([]rune(blocklistItem)) > 240 {
+	)
+
+	// remove duplicates
+	domains = removeDuplicateStr(domains)
+
+	for _, domain := range domains {
+		// removing trailing dots
+		domain = regexp.MustCompile(`\.$`).ReplaceAllString(domain, "")
+
+		// skip domains that are too long
+		if len([]rune(domain)) > 230 {
 			continue
 		}
-		list = append(list, blocklistItem)
-	}
-	domains = list
-	log.Info("hosts removed from blocklist", "reason", "too many characters", "hosts", total-len(domains))
 
-	// remove allow-listed matches
+		// skip domains that are allowed
-	total = len(domains)
+		for _, allowRegex := range cfg.ConfigFile.AllowLists {
-
+			if regexp.MustCompile(allowRegex).MatchString(domain) {
-	// filter out bad regex
-	goodAllowedItemList := []string{}
-	for _, allowedItem := range cfg.ConfigFile.AllowLists {
-		_, err := regexp.Compile(allowedItem)
-		if err != nil {
-			log.Error("unable to parse allow list item", "error", err, "regex", allowedItem)
 				continue
 			}
-		goodAllowedItemList = append(goodAllowedItemList, allowedItem)
 		}
 
-	list = []string{}
+		// add domain
-	for _, v := range domains {
+		cleanDomains = append(cleanDomains, domain)
-		addEntry := true
-		for _, allowedItem := range goodAllowedItemList {
-			if regexp.MustCompile(allowedItem).MatchString(v) {
-				log.Debug("hosts removed from blocklist", "reason", "allowed host", "match string", allowedItem, "host", v)
-				addEntry = false
-			}
-		}
-		if addEntry {
-			list = append(list, v)
-		}
-	}
-	domains = list
-	log.Info("hosts removed from blocklist", "hosts", total-len(domains))
 
-	log.Info("total domains in list", "hosts", len(domains))
+		if t%10000 == 0 {
-	sort.Strings(domains)
+			log.Debug("Clean-Up", "number", t)
-	return domains
+		}
+		t++
+	}
+
+	sort.Strings(cleanDomains)
+	return cleanDomains
 }

cmd/bind/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"os"
+	"regexp"
 	"time"
 
 	"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/config"
@@ -21,6 +22,15 @@ func main() {
 		os.Exit(1)
 	}
 
+	// test allow regex
+	for _, allowedItem := range cfg.ConfigFile.AllowLists {
+		_, err := regexp.Compile(allowedItem)
+		if err != nil {
+			log.Error("unable to parse allow list item", "error", err, "regex", allowedItem)
+			os.Exit(1)
+		}
+	}
+
 	// Set the zone serial number
 	cfg.ConfigFile.ZoneConfig.Serial = time.Now().In(cfg.TZLocal).Format("0601021504")