some library updates and adds support for adblock lists

This commit is contained in:
2025-05-02 20:57:03 -05:00
parent 03b1cc13ee
commit ce4b4a11ff
19 changed files with 712 additions and 139 deletions

View File

@@ -1,4 +1,4 @@
package config
package common
import (
"fmt"

View File

@@ -18,7 +18,7 @@ type structInfo struct {
Tags reflect.StructTag
Type reflect.Type
DefaultValue interface{}
Secret interface{}
Secret bool
}
func getEnv[t string | bool | int | int64 | float64](env string, def t) (t, error) {
@@ -77,6 +77,10 @@ func getStructInfo(spec interface{}) ([]structInfo, error) {
}
typeOfSpec := s.Type()
return parseStructInfo(s, typeOfSpec)
}
func parseStructInfo(s reflect.Value, typeOfSpec reflect.Type) ([]structInfo, error) {
infos := make([]structInfo, 0, s.NumField())
for i := 0; i < s.NumField(); i++ {
f := s.Field(i)
@@ -87,17 +91,7 @@ func getStructInfo(spec interface{}) ([]structInfo, error) {
continue
}
for f.Kind() == reflect.Pointer {
if f.IsNil() {
if f.Type().Elem().Kind() != reflect.Struct {
break
}
f.Set(reflect.New(f.Type().Elem()))
}
f = f.Elem()
}
secret, err := typeConversion(ftype.Type.String(), ftype.Tag.Get("secret"))
secret, err := strconv.ParseBool(ftype.Tag.Get("secret"))
if err != nil {
secret = false
}
@@ -110,31 +104,41 @@ func getStructInfo(spec interface{}) ([]structInfo, error) {
}
info := structInfo{
Name: ftype.Name,
Alt: strings.ToUpper(ftype.Tag.Get("env")),
Info: desc,
Key: ftype.Name,
Field: f,
Tags: ftype.Tag,
Type: ftype.Type,
Secret: secret,
}
if info.Alt != "" {
info.Key = info.Alt
}
info.Key = strings.ToUpper(info.Key)
if ftype.Tag.Get("default") != "" {
v, err := typeConversion(ftype.Type.String(), ftype.Tag.Get("default"))
if err != nil {
return []structInfo{}, err
}
info.DefaultValue = v
Alt: strings.ToUpper(ftype.Tag.Get("env")),
DefaultValue: getDefault(ftype),
Field: f,
Info: desc,
Key: getAlt(ftype),
Name: ftype.Name,
Secret: secret,
Tags: ftype.Tag,
Type: ftype.Type,
}
infos = append(infos, info)
}
return infos, nil
}
func getAlt(ftype reflect.StructField) string {
if len(ftype.Tag.Get("env")) > 0 {
return strings.ToUpper(ftype.Tag.Get("env"))
}
return strings.ToUpper(ftype.Name)
}
func getDefault(ftype reflect.StructField) interface{} {
if ftype.Tag.Get("default") != "" {
v, err := typeConversion(ftype.Type.String(), ftype.Tag.Get("default"))
if err != nil {
return nil
}
return v
}
return nil
}
func typeConversion(t, v string) (interface{}, error) {
switch t {
case "string": //nolint:goconst

View File

@@ -0,0 +1,191 @@
package config
import (
"strconv"
"testing"
"github.com/stretchr/testify/assert"
)
type mock_config struct {
NoTags string
Ignored string `ignored:"true"`
Info string `info:"This is an info string."`
Secret string `secret:"true"`
Env string `env:"test_env"`
Default_string string `default:"This is a default string."`
Default_bool bool `default:"true"`
Default_int int `default:"100"`
Default_int64 int64 `default:"100"`
Default_float64 float64 `default:"100.001"`
}
func TestGetEnv(t *testing.T) {
var (
expected_string string = "This is a default string."
expected_bool bool = true
expected_int int = 100
expected_int64 int64 = 100
expected_float64 float64 = 100.001
expected_unset_default string = "This is a default value."
)
// string
t.Setenv("TEST_STRING", expected_string)
test_string, err := getEnv("TEST_STRING", "This is a default string.")
assert.NoError(t, err)
assert.Equal(t, expected_string, test_string)
// bool
_, err = getEnv("TEST_STRING", expected_bool)
assert.Error(t, err)
t.Setenv("TEST_BOOL", strconv.FormatBool(expected_bool))
test_bool, err := getEnv("TEST_BOOL", expected_bool)
assert.NoError(t, err)
assert.Equal(t, expected_bool, test_bool)
// int
_, err = getEnv("TEST_STRING", expected_int)
assert.Error(t, err)
t.Setenv("TEST_INT", strconv.FormatInt(int64(expected_int), 10))
test_int, err := getEnv("TEST_INT", expected_int)
assert.NoError(t, err)
assert.Equal(t, expected_int, test_int)
// int64
_, err = getEnv("TEST_STRING", expected_int64)
assert.Error(t, err)
t.Setenv("TEST_INT64", strconv.FormatInt(expected_int64, 10))
test_int64, err := getEnv("TEST_INT", expected_int64)
assert.NoError(t, err)
assert.Equal(t, expected_int64, test_int64)
// float64
_, err = getEnv("TEST_STRING", expected_float64)
assert.Error(t, err)
t.Setenv("TEST_INT", strconv.FormatFloat(expected_float64, 'f', 3, 64))
test_float64, err := getEnv("TEST_INT", expected_float64)
assert.NoError(t, err)
assert.Equal(t, expected_float64, test_float64)
// unset or missing environment variable
test_unset, err := getEnv("TEST_DEFAULT", expected_unset_default)
assert.NoError(t, err)
assert.Equal(t, expected_unset_default, test_unset)
}
func TestGetStructInfo(t *testing.T) {
test_config := mock_config{
NoTags: "notags",
Ignored: "ignored",
Secret: "secret",
}
cfgInfo, err := getStructInfo(&test_config)
assert.NoError(t, err)
for _, v := range cfgInfo {
switch v.Name {
case "Info":
assert.Equal(t, "() This is an info string.", v.Info)
case "Secret":
assert.Equal(t, true, v.Secret)
case "Env":
assert.Equal(t, "TEST_ENV", v.Alt)
case "Default_value":
assert.Equal(t, "This is a default string.", v.DefaultValue)
}
}
}
func TestTypeConversion(t *testing.T) {
var (
expected_string string = "This is a default string."
expected_int int = 100
expected_int8 int8 = 100
expected_int16 int16 = 100
expected_int32 int32 = 100
expected_int64 int64 = 100
expected_uint uint = 100
expected_uint16 uint16 = 100
expected_uint32 uint32 = 100
expected_uint64 uint64 = 100
expected_float32 float32 = 100.001
expected_float64 float64 = 100.001
expected_bool bool = true
)
// string
output_string, err := typeConversion("string", expected_string)
assert.NoError(t, err)
assert.Equal(t, expected_string, output_string)
// int
output_int, err := typeConversion("int", strconv.FormatInt(int64(expected_int), 10))
assert.NoError(t, err)
assert.Equal(t, expected_int, int(output_int.(int64)))
// int8
output_int8, err := typeConversion("int8", strconv.FormatInt(int64(expected_int8), 10))
assert.NoError(t, err)
assert.Equal(t, expected_int8, int8(output_int8.(int64))) // nolint: gosec
// int16
output_int16, err := typeConversion("int16", strconv.FormatInt(int64(expected_int16), 10))
assert.NoError(t, err)
assert.Equal(t, expected_int16, int16(output_int16.(int64))) // nolint: gosec
// int32
output_int32, err := typeConversion("int32", strconv.FormatInt(int64(expected_int32), 10))
assert.NoError(t, err)
assert.Equal(t, expected_int32, int32(output_int32.(int64))) // nolint: gosec
// int64
output_int64, err := typeConversion("int64", strconv.FormatInt(expected_int64, 10))
assert.NoError(t, err)
assert.Equal(t, expected_int64, output_int64)
// uint
output_uint, err := typeConversion("uint", strconv.FormatInt(int64(expected_uint), 10))
assert.NoError(t, err)
assert.Equal(t, expected_uint, uint(output_uint.(uint64))) // nolint: gosec
// uint16
output_uint16, err := typeConversion("uint16", strconv.FormatInt(int64(expected_uint16), 10))
assert.NoError(t, err)
assert.Equal(t, expected_uint16, uint16(output_uint16.(uint64))) // nolint: gosec
// uint32
output_uint32, err := typeConversion("uint32", strconv.FormatInt(int64(expected_uint32), 10))
assert.NoError(t, err)
assert.Equal(t, expected_uint32, uint32(output_uint32.(uint64))) // nolint: gosec
// uint64
output_uint64, err := typeConversion("uint64", strconv.FormatInt(int64(expected_uint64), 10))
assert.NoError(t, err)
assert.Equal(t, expected_uint64, output_uint64)
// float32
output_float32, err := typeConversion("float32", strconv.FormatFloat(float64(expected_float32), 'f', 3, 64))
assert.NoError(t, err)
assert.Equal(t, expected_float32, float32(output_float32.(float64))) // nolint: gosec
// float64
output_float64, err := typeConversion("float64", strconv.FormatFloat(expected_float64, 'f', 3, 64))
assert.NoError(t, err)
assert.Equal(t, expected_float64, output_float64)
// bool
output_bool, err := typeConversion("bool", strconv.FormatBool(expected_bool))
assert.NoError(t, err)
assert.Equal(t, expected_bool, output_bool)
}
func TestParseFlags(t *testing.T) {
test_config := Config{}
cfgInfo, err := getStructInfo(&test_config)
assert.NoError(t, err)
assert.NoError(t, test_config.parseFlags(cfgInfo))
}

View File

@@ -2,63 +2,62 @@ package config
import (
"fmt"
"os"
"pihole-blocklist/bind/assets"
"time"
"gopkg.in/yaml.v3"
"github.com/goccy/go-yaml"
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/assets"
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/common"
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/log"
)
func Init() Config {
func Init() (Config, error) {
cfg := New()
// parse config structure
cfgInfo, err := getStructInfo(&cfg)
if err != nil {
panic(fmt.Sprintf("Unable to initialize program: %v", err))
return Config{}, err
}
// get command line flags
if err := cfg.parseFlags(cfgInfo); err != nil {
panic(fmt.Sprintf("Unable to initialize program: %v", err))
return Config{}, err
}
// set logging Level
setLogLevel(&cfg)
log.Init("text")
log.SetNumericLevel(cfg.LogLevel)
// set timezone & time format
cfg.TZUTC, _ = time.LoadLocation("UTC")
cfg.TZLocal, err = time.LoadLocation(cfg.TimeZoneLocal)
if err != nil {
cfg.Log.Error("Unable to parse timezone string", "error", err)
os.Exit(1)
return Config{}, err
}
// check config file
if !FileExists(cfg.ConfigFileLocation) {
if _, err := WriteFile(cfg.ConfigFileLocation, assets.Config); err != nil {
cfg.Log.Error(err.Error())
os.Exit(1)
if !common.FileExists(cfg.ConfigFileLocation) {
if _, err := common.WriteFile(cfg.ConfigFileLocation, assets.Config); err != nil {
return Config{}, err
}
cfg.Log.Error("Unable to locate configuration file, an example config file has been written", "path", cfg.ConfigFileLocation)
os.Exit(1)
return Config{}, fmt.Errorf("Unable to locate configuration file, an example config file has been written to %s", cfg.ConfigFileLocation)
}
// read config
cfData, err := ReadFile(cfg.ConfigFileLocation)
cfData, err := common.ReadFile(cfg.ConfigFileLocation)
if err != nil {
cfg.Log.Error("Unable to read config file", "error", err)
os.Exit(1)
return Config{}, err
}
// unmarshal config file
if err := yaml.Unmarshal(cfData, &cfg.ConfigFile); err != nil {
cfg.Log.Error("Unable to read config file contents", "error", err)
os.Exit(1)
return Config{}, err
}
// print running config
printRunningConfig(&cfg, cfgInfo)
// return configuration
return cfg
return cfg, nil
}

View File

@@ -1,13 +1,19 @@
package config
import (
"log/slog"
"os"
"reflect"
"strconv"
"time"
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/log"
)
// Config uses struct tags to configure the application.
// (default) Default value to be used if unset or not defined.
// (ignored) Don't process the current tag.
// (info) String to be presented to the user on -help use.
// (secret) If set to true, hide the value from being output on start-up.
// (env) environment variable to be used if not set on command line.
type Config struct {
// time configuration
TimeFormat string `default:"2006-01-02 15:04:05" env:"time_format"`
@@ -16,21 +22,20 @@ type Config struct {
TZUTC *time.Location `ignored:"true"`
// logging
LogLevel int `default:"50" env:"log_level"`
Log *slog.Logger `ignored:"true"`
SLogLevel *slog.LevelVar `ignored:"true"`
LogLevel int `default:"50" env:"log_level"`
// HTTP Client timeout configurations
HTTPClientRequestTimeout int `default:"60" env:"HTTP_CLIENT_REQUEST_TIMEOUT"`
HTTPClientConnectTimeout int `default:"5" env:"HTTP_CLIENT_CONNECT_TIMEOUT"`
HTTPClientTLSHandshakeTimeout int `default:"5" env:"HTTP_CLIENT_TLS_TIMEOUT"`
HTTPClientIdleTimeout int `default:"5" env:"HTTP_CLIENT_IDLE_TIMEOUT"`
// webserver
WebServerPort int `default:"8080" env:"webserver_port"`
WebServerIP string `default:"0.0.0.0" env:"webserver_ip"`
WebServerReadTimeout int `default:"5" env:"webserver_read_timeout"`
WebServerWriteTimeout int `default:"1" env:"webserver_write_timeout"`
WebServerIdleTimeout int `default:"2" env:"webserver_idle_timeout"`
// Output Filename
BindOutputFileName string `default:"./response-policy.bind" env:"OUTPUT"`
BindOutputFileName string `default:"./response-policy.bind" env:"output"`
// Config
ConfigFileLocation string `default:"./config.yaml" env:"CONFIG_FILE"`
ConfigFileLocation string `default:"./config.yaml" env:"config_file"`
ConfigFile configFileStruct
}
@@ -48,6 +53,7 @@ type configFileStruct struct {
TTL string `yaml:"timeToLive"`
} `yaml:"zoneConfig"`
Sources struct {
AdBlockURLs []string `yaml:"adBlockURLs"`
DomainListURLs []string `yaml:"domainListURLs"`
HostFileURLs []string `yaml:"hostFileURLs"`
} `yaml:"sources"`
@@ -57,52 +63,27 @@ type configFileStruct struct {
// New initializes the config variable for use with a prepared set of defaults.
func New() Config {
cfg := Config{
SLogLevel: new(slog.LevelVar),
}
cfg.Log = slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
Level: cfg.SLogLevel,
}))
return cfg
}
func setLogLevel(cfg *Config) {
switch {
// error
case cfg.LogLevel <= 20:
cfg.SLogLevel.Set(slog.LevelError)
cfg.Log.Info("Log level updated", "level", slog.LevelError)
// warning
case cfg.LogLevel > 20 && cfg.LogLevel <= 40:
cfg.SLogLevel.Set(slog.LevelWarn)
cfg.Log.Info("Log level updated", "level", slog.LevelWarn)
// info
case cfg.LogLevel > 40 && cfg.LogLevel <= 60:
cfg.SLogLevel.Set(slog.LevelInfo)
cfg.Log.Info("Log level updated", "level", slog.LevelInfo)
// debug
case cfg.LogLevel > 60:
cfg.SLogLevel.Set(slog.LevelDebug)
cfg.Log.Info("Log level updated", "level", slog.LevelDebug)
}
// set default logger
slog.SetDefault(cfg.Log)
return Config{}
}
func printRunningConfig(cfg *Config, cfgInfo []structInfo) {
var logRunningConfiguration string = "Running Configuration"
for _, info := range cfgInfo {
switch info.Type.String() {
case "string":
p := reflect.ValueOf(cfg).Elem().FieldByName(info.Name).Addr().Interface().(*string)
cfg.Log.Debug("Running Configuration", info.Alt, *p)
case "bool":
p := reflect.ValueOf(cfg).Elem().FieldByName(info.Name).Addr().Interface().(*bool)
cfg.Log.Debug("Running Configuration", info.Alt, strconv.FormatBool(*p))
case "int":
p := reflect.ValueOf(cfg).Elem().FieldByName(info.Name).Addr().Interface().(*int)
cfg.Log.Debug("Running Configuration", info.Alt, strconv.FormatInt(int64(*p), 10))
if info.Secret {
log.Debug(logRunningConfiguration, info.Name, "REDACTED")
} else {
switch info.Type.String() {
case "string":
p := reflect.ValueOf(cfg).Elem().FieldByName(info.Name).Addr().Interface().(*string)
log.Debug(logRunningConfiguration, info.Alt, *p)
case "bool":
p := reflect.ValueOf(cfg).Elem().FieldByName(info.Name).Addr().Interface().(*bool)
log.Debug(logRunningConfiguration, info.Alt, strconv.FormatBool(*p))
case "int":
p := reflect.ValueOf(cfg).Elem().FieldByName(info.Name).Addr().Interface().(*int)
log.Debug(logRunningConfiguration, info.Alt, strconv.FormatInt(int64(*p), 10))
}
}
}
}

View File

@@ -0,0 +1,40 @@
package config
import (
"bytes"
"log/slog"
"reflect"
"testing"
"github.com/stretchr/testify/assert"
"gitlab.smoothnet.org/nhyatt/bind-response-policy-zone-creator/internal/log"
)
func slogToBuffer() (*bytes.Buffer, *slog.Logger) {
buf := new(bytes.Buffer)
return buf, slog.New(
slog.NewTextHandler(
buf,
&slog.HandlerOptions{
Level: log.LevelTrace,
},
),
)
}
func TestPrintRunningConfig(t *testing.T) {
buf, l := slogToBuffer()
log.L.Log = l
c := New()
cfgInfo, err := getStructInfo(&c)
assert.NoError(t, err)
printRunningConfig(&c, cfgInfo)
assert.Contains(t, buf.String(), "Running Configuration")
}
func TestNew(t *testing.T) {
c := New()
assert.Equal(t, "config.Config", reflect.TypeOf(c).String())
}

154
internal/log/logging.go Normal file
View File

@@ -0,0 +1,154 @@
package log
import (
"context"
"log/slog"
"os"
)
const (
LevelTrace = slog.Level(-8)
LevelFatal = slog.Level(12)
)
type Log struct {
Ctx context.Context
Log *slog.Logger
SLogLevel slog.LevelVar
}
var (
// LevelNames set the names associated with custom logging levels.
LevelNames = map[slog.Leveler]string{
LevelTrace: "TRACE",
LevelFatal: "FATAL",
}
// L is the global interface used for calling the logger subfunctions.
L = Log{}
)
func Init(writer string) {
slogOptions := &slog.HandlerOptions{
Level: &L.SLogLevel,
ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr {
if a.Key == slog.TimeKey {
a.Value = slog.StringValue(a.Value.Time().Format("2006-01-02T15:04:05.000-0700"))
a.Key = "ts"
}
if a.Key == slog.LevelKey {
level := a.Value.Any().(slog.Level)
levelLabel, exists := LevelNames[level]
if !exists {
levelLabel = level.String()
}
a.Value = slog.StringValue(levelLabel)
}
return a
},
}
// Initialize SLog and translate new logging levels
switch writer {
case "json":
L.Log = slog.New(slog.NewJSONHandler(os.Stdout, slogOptions))
default:
L.Log = slog.New(slog.NewTextHandler(os.Stdout, slogOptions))
}
// create context
L.Ctx = context.Background()
}
// SetNumericLevel will set the log level based on a number from 1-100.
// The larger the number the more verbose the logs.
//
// 1-20 = Fatal, 21-40 = Error, 41-60 = Warn, 61-80 = Info, 81-99 = Debug,
// and 100 = Trace.
func SetNumericLevel(level int) {
var llu string = "Log Level Updated"
switch {
// fatal
case level <= 20:
L.SLogLevel.Set(LevelFatal)
Info(llu, "level", LevelFatal)
// error
case level > 20 && level <= 40:
L.SLogLevel.Set(slog.LevelError)
Info(llu, "level", slog.LevelError)
// warning
case level > 40 && level <= 60:
L.SLogLevel.Set(slog.LevelWarn)
Info(llu, "level", slog.LevelWarn)
// info
case level > 60 && level <= 80:
L.SLogLevel.Set(slog.LevelInfo)
Info(llu, "level", slog.LevelInfo)
// debug
case level > 80 && level <= 99:
L.SLogLevel.Set(slog.LevelDebug)
Info(llu, "level", slog.LevelDebug)
// trace
case level > 99:
L.SLogLevel.Set(LevelTrace)
Info(llu, "level", LevelTrace)
}
// set default logger
slog.SetDefault(L.Log)
}
func Fatal(msg string, attrs ...interface{}) {
L.Log.Log(
L.Ctx,
LevelFatal,
msg,
attrs...,
)
}
func Error(msg string, attrs ...interface{}) {
L.Log.Log(
L.Ctx,
slog.LevelError,
msg,
attrs...,
)
}
func Warn(msg string, attrs ...interface{}) {
L.Log.Log(
L.Ctx,
slog.LevelWarn,
msg,
attrs...,
)
}
func Info(msg string, attrs ...interface{}) {
L.Log.Log(
L.Ctx,
slog.LevelInfo,
msg,
attrs...,
)
}
func Debug(msg string, attrs ...interface{}) {
L.Log.Log(
L.Ctx,
slog.LevelDebug,
msg,
attrs...,
)
}
func Trace(msg string, attrs ...interface{}) {
L.Log.Log(
L.Ctx,
LevelTrace,
msg,
attrs...,
)
}

View File

@@ -0,0 +1,97 @@
package log
import (
"bytes"
"log/slog"
"testing"
"github.com/stretchr/testify/assert"
)
func slogToBuffer() (*bytes.Buffer, *slog.Logger) {
buf := new(bytes.Buffer)
return buf, slog.New(
slog.NewTextHandler(
buf,
&slog.HandlerOptions{
Level: LevelTrace,
},
),
)
}
func TestSetLogLevel(t *testing.T) {
Init("text")
for _, i := range []int{0, 21, 41, 61, 81, 101} {
SetNumericLevel(i)
switch i {
case 0:
assert.Equal(t, LevelFatal, L.SLogLevel.Level())
case 21:
assert.Equal(t, slog.LevelError, L.SLogLevel.Level())
case 41:
assert.Equal(t, slog.LevelWarn, L.SLogLevel.Level())
case 61:
assert.Equal(t, slog.LevelInfo, L.SLogLevel.Level())
case 81:
assert.Equal(t, slog.LevelDebug, L.SLogLevel.Level())
case 101:
assert.Equal(t, LevelTrace, L.SLogLevel.Level())
}
}
}
func TestFatal(t *testing.T) {
buf, log := slogToBuffer()
L.Log = log
Fatal("TEST Message")
assert.Contains(t, buf.String(), "TEST Message")
assert.Contains(t, buf.String(), "level=ERROR+4")
}
func TestError(t *testing.T) {
buf, log := slogToBuffer()
L.Log = log
Error("TEST Message")
assert.Contains(t, buf.String(), "TEST Message")
assert.Contains(t, buf.String(), "level=ERROR")
}
func TestWarn(t *testing.T) {
buf, log := slogToBuffer()
L.Log = log
Warn("TEST Message")
assert.Contains(t, buf.String(), "TEST Message")
assert.Contains(t, buf.String(), "level=WARN")
}
func TestInfo(t *testing.T) {
buf, log := slogToBuffer()
L.Log = log
Info("TEST Message")
assert.Contains(t, buf.String(), "TEST Message")
assert.Contains(t, buf.String(), "level=INFO")
}
func TestDebug(t *testing.T) {
buf, log := slogToBuffer()
L.Log = log
Debug("TEST Message")
assert.Contains(t, buf.String(), "TEST Message")
assert.Contains(t, buf.String(), "level=DEBUG")
}
func TestTrace(t *testing.T) {
buf, log := slogToBuffer()
L.Log = log
Trace("TEST Message")
assert.Contains(t, buf.String(), "TEST Message")
assert.Contains(t, buf.String(), "level=DEBUG-4")
}