multiple updates

This commit is contained in:
2023-12-09 14:21:28 -06:00
parent a11f92745d
commit 881c11b910
23 changed files with 721 additions and 433 deletions

View File

@@ -1,15 +1,11 @@
package main
import (
"log"
"regexp"
"sort"
"time"
)
func cleanBadDomains(domains []string) []string {
defer timeTrack(time.Now(), "cleanBadDomains")
// remove duplicates
total := len(domains)
all := make(map[string]bool)
@@ -21,7 +17,7 @@ func cleanBadDomains(domains []string) []string {
}
}
domains = list
log.Printf("[INFO] Duplicate items removed: %d\n", total-len(domains))
cfg.Log.Info("hosts removed from blocklist", "reason", "duplicate", "hosts", total-len(domains))
// remove hosts that are too long
total = len(domains)
@@ -33,18 +29,18 @@ func cleanBadDomains(domains []string) []string {
list = append(list, blocklistItem)
}
domains = list
log.Printf("[INFO] Hosts with too many characters removed: %d\n", total-len(domains))
cfg.Log.Info("hosts removed from blocklist", "reason", "too many characters", "hosts", total-len(domains))
// remove allow-listed matches
total = len(domains)
// filter out bad regex
goodAllowedItemList := []string{}
for _, allowedItem := range config.Config.AllowLists {
for _, allowedItem := range cfg.ConfigFile.AllowLists {
_, err := regexp.Compile(allowedItem)
if err != nil {
log.Printf("[ERROR] Allow list item (%s) is not valid regex: %v\n", allowedItem, err)
break
cfg.Log.Error("unable to parse allow list item", "error", err, "regex", allowedItem)
continue
}
goodAllowedItemList = append(goodAllowedItemList, allowedItem)
}
@@ -54,7 +50,7 @@ func cleanBadDomains(domains []string) []string {
addEntry := true
for _, allowedItem := range goodAllowedItemList {
if regexp.MustCompile(allowedItem).MatchString(v) {
log.Printf("[DEBUG] Removing allowed matching item: %s\n", v)
cfg.Log.Debug("hosts removed from blocklist", "reason", "allowed host", "match string", allowedItem, "host", v)
addEntry = false
}
}
@@ -63,9 +59,9 @@ func cleanBadDomains(domains []string) []string {
}
}
domains = list
log.Printf("[INFO] Allowed hosts removed: %d\n", total-len(domains))
cfg.Log.Info("hosts removed from blocklist", "hosts", total-len(domains))
log.Printf("[INFO] Total domains in list at end: %d.\n", len(domains))
cfg.Log.Info("total domains in list", "hosts", len(domains))
sort.Strings(domains)
return domains
}