improve performace?

cmd/bind/cleanup.go

@@ -7,63 +7,52 @@ import (
 	"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/log"
 )
 
-func cleanBadDomains(domains []string) []string {
-	// remove duplicates
-	total := len(domains)
-	all := make(map[string]bool)
-	list := []string{}
+func removeDuplicateStr(domains []string) []string {
+	allKeys := make(map[string]bool)
+	allDomains := []string{}
 	for _, item := range domains {
-		if _, value := all[item]; !value {
-			all[item] = true
-			list = append(list, item)
+		if _, value := allKeys[item]; !value {
+			allKeys[item] = true
+			allDomains = append(allDomains, item)
 		}
 	}
-	domains = list
-	log.Info("hosts removed from blocklist", "reason", "duplicate", "hosts", total-len(domains))
+	return allDomains
+}
 
-	// remove hosts that are too long
-	total = len(domains)
-	list = []string{}
-	for _, blocklistItem := range domains {
-		if len([]rune(blocklistItem)) > 240 {
+func cleanBadDomains(domains []string) []string {
+	var (
+		cleanDomains []string
+		t            int64
+	)
+
+	// remove duplicates
+	domains = removeDuplicateStr(domains)
+
+	for _, domain := range domains {
+		// removing trailing dots
+		domain = regexp.MustCompile(`\.$`).ReplaceAllString(domain, "")
+
+		// skip domains that are too long
+		if len([]rune(domain)) > 230 {
 			continue
 		}
-		list = append(list, blocklistItem)
-	}
-	domains = list
-	log.Info("hosts removed from blocklist", "reason", "too many characters", "hosts", total-len(domains))
 
-	// remove allow-listed matches
-	total = len(domains)
-
-	// filter out bad regex
-	goodAllowedItemList := []string{}
-	for _, allowedItem := range cfg.ConfigFile.AllowLists {
-		_, err := regexp.Compile(allowedItem)
-		if err != nil {
-			log.Error("unable to parse allow list item", "error", err, "regex", allowedItem)
-			continue
-		}
-		goodAllowedItemList = append(goodAllowedItemList, allowedItem)
-	}
-
-	list = []string{}
-	for _, v := range domains {
-		addEntry := true
-		for _, allowedItem := range goodAllowedItemList {
-			if regexp.MustCompile(allowedItem).MatchString(v) {
-				log.Debug("hosts removed from blocklist", "reason", "allowed host", "match string", allowedItem, "host", v)
-				addEntry = false
+		// skip domains that are allowed
+		for _, allowRegex := range cfg.ConfigFile.AllowLists {
+			if regexp.MustCompile(allowRegex).MatchString(domain) {
+				continue
 			}
 		}
-		if addEntry {
-			list = append(list, v)
-		}
-	}
-	domains = list
-	log.Info("hosts removed from blocklist", "hosts", total-len(domains))
 
-	log.Info("total domains in list", "hosts", len(domains))
-	sort.Strings(domains)
-	return domains
+		// add domain
+		cleanDomains = append(cleanDomains, domain)
+
+		if t%10000 == 0 {
+			log.Debug("Clean-Up", "number", t)
+		}
+		t++
+	}
+
+	sort.Strings(cleanDomains)
+	return cleanDomains
 }