Merge pull request #259 from folbricht/issue-153

Allow blocklist loader to fail on startup

blocklistdb-domain.go

@@ -1,118 +1,118 @@
-package rdns
-
-import (
-	"fmt"
-	"net"
-	"strings"
-
-	"github.com/miekg/dns"
-)
-
-// DomainDB holds a list of domain strings (potentially with wildcards). Matching
-// logic:
-// domain.com: matches just domain.com and not subdomains
-// .domain.com: matches domain.com and all subdomains
-// *.domain.com: matches all subdomains but not domain.com
-type DomainDB struct {
-	name   string
-	root   node
-	loader BlocklistLoader
-}
-
-type node map[string]node
-
-var _ BlocklistDB = &DomainDB{}
-
-// NewDomainDB returns a new instance of a matcher for a list of regular expressions.
-func NewDomainDB(name string, loader BlocklistLoader) (*DomainDB, error) {
-	rules, err := loader.Load()
-	if err != nil {
-		return nil, err
-	}
-	root := make(node)
-	for _, r := range rules {
-		r = strings.TrimSpace(r)
-
-		// Strip trailing . in case the list has FQDN names with . suffixes.
-		r = strings.TrimSuffix(r, ".")
-
-		// Break up the domain into its parts and iterare backwards over them, building
-		// a graph of maps
-		parts := strings.Split(r, ".")
-		n := root
-		for i := len(parts) - 1; i >= 0; i-- {
-			part := parts[i]
-
-			// Only allow wildcards as the first domain part, and not in a string
-			if strings.Contains(part, "*") && (i > 0 || len(part) != 1) {
-				return nil, fmt.Errorf("invalid blocklist item: '%s'", part)
-			}
-
-			subNode, ok := n[part]
-			if !ok {
-				subNode = make(node)
-				n[part] = subNode
-			}
-			n = subNode
-		}
-	}
-	return &DomainDB{name, root, loader}, nil
-}
-
-func (m *DomainDB) Reload() (BlocklistDB, error) {
-	return NewDomainDB(m.name, m.loader)
-}
-
-func (m *DomainDB) Match(q dns.Question) (net.IP, string, *BlocklistMatch, bool) {
-	s := strings.TrimSuffix(q.Name, ".")
-	var matched []string
-	parts := strings.Split(s, ".")
-	n := m.root
-	for i := len(parts) - 1; i >= 0; i-- {
-		part := parts[i]
-		subNode, ok := n[part]
-		if !ok {
-			return nil, "", nil, false
-		}
-		matched = append(matched, part)
-		if _, ok := subNode[""]; ok { // exact and sub-domain match
-			return nil,
-				"",
-				&BlocklistMatch{
-					List: m.name,
-					Rule: matchedDomainParts(".", matched),
-				},
-				true
-		}
-		if _, ok := subNode["*"]; ok && i > 0 { // wildcard match on sub-domains
-			return nil,
-				"",
-				&BlocklistMatch{
-					List: m.name,
-					Rule: matchedDomainParts("*.", matched),
-				},
-				true
-		}
-		n = subNode
-	}
-	return nil,
-		"",
-		&BlocklistMatch{
-			List: m.name,
-			Rule: matchedDomainParts("", matched),
-		},
-		len(n) == 0 // exact match
-}
-
-func (m *DomainDB) String() string {
-	return "Domain"
-}
-
-// Turn a list of matched domain fragments into a domain (rule)
-func matchedDomainParts(prefix string, p []string) string {
-	for i := len(p)/2 - 1; i >= 0; i-- {
-		opp := len(p) - 1 - i
-		p[i], p[opp] = p[opp], p[i]
-	}
-	return prefix + strings.Join(p, ".")
-}
+package rdns
+
+import (
+	"fmt"
+	"net"
+	"strings"
+
+	"github.com/miekg/dns"
+)
+
+// DomainDB holds a list of domain strings (potentially with wildcards). Matching
+// logic:
+// domain.com: matches just domain.com and not subdomains
+// .domain.com: matches domain.com and all subdomains
+// *.domain.com: matches all subdomains but not domain.com
+type DomainDB struct {
+	name   string
+	root   node
+	loader BlocklistLoader
+}
+
+type node map[string]node
+
+var _ BlocklistDB = &DomainDB{}
+
+// NewDomainDB returns a new instance of a matcher for a list of regular expressions.
+func NewDomainDB(name string, loader BlocklistLoader) *DomainDB {
+	return &DomainDB{name, nil, loader}
+}
+
+func (m *DomainDB) Reload() (BlocklistDB, error) {
+	rules, err := m.loader.Load()
+	if err != nil {
+		return nil, err
+	}
+	root := make(node)
+	for _, r := range rules {
+		r = strings.TrimSpace(r)
+
+		// Strip trailing . in case the list has FQDN names with . suffixes.
+		r = strings.TrimSuffix(r, ".")
+
+		// Break up the domain into its parts and iterate backwards over them, building
+		// a graph of maps
+		parts := strings.Split(r, ".")
+		n := root
+		for i := len(parts) - 1; i >= 0; i-- {
+			part := parts[i]
+
+			// Only allow wildcards as the first domain part, and not in a string
+			if strings.Contains(part, "*") && (i > 0 || len(part) != 1) {
+				return nil, fmt.Errorf("invalid blocklist item: '%s'", part)
+			}
+
+			subNode, ok := n[part]
+			if !ok {
+				subNode = make(node)
+				n[part] = subNode
+			}
+			n = subNode
+		}
+	}
+	return &DomainDB{m.name, root, m.loader}, nil
+}
+
+func (m *DomainDB) Match(q dns.Question) (net.IP, string, *BlocklistMatch, bool) {
+	s := strings.TrimSuffix(q.Name, ".")
+	var matched []string
+	parts := strings.Split(s, ".")
+	n := m.root
+	for i := len(parts) - 1; i >= 0; i-- {
+		part := parts[i]
+		subNode, ok := n[part]
+		if !ok {
+			return nil, "", nil, false
+		}
+		matched = append(matched, part)
+		if _, ok := subNode[""]; ok { // exact and sub-domain match
+			return nil,
+				"",
+				&BlocklistMatch{
+					List: m.name,
+					Rule: matchedDomainParts(".", matched),
+				},
+				true
+		}
+		if _, ok := subNode["*"]; ok && i > 0 { // wildcard match on sub-domains
+			return nil,
+				"",
+				&BlocklistMatch{
+					List: m.name,
+					Rule: matchedDomainParts("*.", matched),
+				},
+				true
+		}
+		n = subNode
+	}
+	return nil,
+		"",
+		&BlocklistMatch{
+			List: m.name,
+			Rule: matchedDomainParts("", matched),
+		},
+		len(n) == 0 // exact match
+}
+
+func (m *DomainDB) String() string {
+	return "Domain"
+}
+
+// Turn a list of matched domain fragments into a domain (rule)
+func matchedDomainParts(prefix string, p []string) string {
+	for i := len(p)/2 - 1; i >= 0; i-- {
+		opp := len(p) - 1 - i
+		p[i], p[opp] = p[opp], p[i]
+	}
+	return prefix + strings.Join(p, ".")
+}

blocklistdb-hosts.go

@@ -25,8 +25,12 @@ type ipRecords struct {
 var _ BlocklistDB = &HostsDB{}
 
 // NewHostsDB returns a new instance of a matcher for a list of regular expressions.
-func NewHostsDB(name string, loader BlocklistLoader) (*HostsDB, error) {
-	rules, err := loader.Load()
+func NewHostsDB(name string, loader BlocklistLoader) *HostsDB {
+	return &HostsDB{name, nil,nil, loader}
+}
+
+func (m *HostsDB) Reload() (BlocklistDB, error) {
+	rules, err := m.loader.Load()
 	if err != nil {
 		return nil, err
 	}
@@ -70,11 +74,8 @@ func NewHostsDB(name string, loader BlocklistLoader) (*HostsDB, error) {
 		}
 		ptrMap[reverseAddr] = names[0]
 	}
-	return &HostsDB{name, filters, ptrMap, loader}, nil
-}
+	return &HostsDB{m.name, filters, ptrMap, m.loader}, nil
 
-func (m *HostsDB) Reload() (BlocklistDB, error) {
-	return NewHostsDB(m.name, m.loader)
 }
 
 func (m *HostsDB) Match(q dns.Question) (net.IP, string, *BlocklistMatch, bool) {

blocklistdb-regexp.go

@@ -18,12 +18,17 @@ type RegexpDB struct {
 var _ BlocklistDB = &RegexpDB{}
 
 // NewRegexpDB returns a new instance of a matcher for a list of regular expressions.
-func NewRegexpDB(name string, loader BlocklistLoader) (*RegexpDB, error) {
-	rules, err := loader.Load()
+func NewRegexpDB(name string, loader BlocklistLoader) *RegexpDB {
+	return &RegexpDB{name, nil, loader}
+}
+
+func (m *RegexpDB) Reload() (BlocklistDB, error) {
+	rules, err := m.loader.Load()
 	if err != nil {
 		return nil, err
 	}
 	var filters []*regexp.Regexp
+
 	for _, r := range rules {
 		r = strings.TrimSpace(r)
 		if r == "" || strings.HasPrefix(r, "#") {
@@ -36,11 +41,7 @@ func NewRegexpDB(name string, loader BlocklistLoader) (*RegexpDB, error) {
 		filters = append(filters, re)
 	}
 
-	return &RegexpDB{name, filters, loader}, nil
-}
-
-func (m *RegexpDB) Reload() (BlocklistDB, error) {
-	return NewRegexpDB(m.name, m.loader)
+	return &RegexpDB{m.name, filters, m.loader}, nil
 }
 
 func (m *RegexpDB) Match(q dns.Question) (net.IP, string, *BlocklistMatch, bool) {

cache.go

@@ -59,7 +59,7 @@ type CacheOptions struct {
 	// Query name that will trigger a cache flush. Disabled if empty.
 	FlushQuery string
 
-	// If a query is received for a record with less that PrefetchTrigger TTL left, the
+	// If a query is received for a record that less that PrefetchTrigger TTL left, the
 	// cache will send another query to upstream. The goal is to automatically refresh
 	// the record in the cache.
 	PrefetchTrigger uint32

cidr-db.go

@@ -17,16 +17,20 @@ type CidrDB struct {
 var _ IPBlocklistDB = &CidrDB{}
 
 // NewCidrDB returns a new instance of a matcher for a list of networks.
-func NewCidrDB(name string, loader BlocklistLoader) (*CidrDB, error) {
-	rules, err := loader.Load()
+func NewCidrDB(name string, loader BlocklistLoader) *CidrDB {
+	return &CidrDB{name, nil, nil, loader}
+}
+
+func (m *CidrDB) Reload() (IPBlocklistDB, error) {
+	rules, err := m.loader.Load()
 	if err != nil {
 		return nil, err
 	}
 	db := &CidrDB{
-		name:   name,
+		name:   m.name,
 		ip4:    new(ipBlocklistTrie),
 		ip6:    new(ipBlocklistTrie),
-		loader: loader,
+		loader: m.loader,
 	}
 	for _, r := range rules {
 		r = strings.TrimSpace(r)
@@ -51,11 +55,7 @@ func NewCidrDB(name string, loader BlocklistLoader) (*CidrDB, error) {
 			db.ip4.add(n)
 		}
 	}
-	return db, nil
-}
-
-func (m *CidrDB) Reload() (IPBlocklistDB, error) {
-	return NewCidrDB(m.name, m.loader)
+	return &CidrDB{m.name, db.ip4, db.ip6, m.loader}, nil
 }
 
 func (m *CidrDB) Match(ip net.IP) (*BlocklistMatch, bool) {

cmd/routedns/config.go

@@ -142,10 +142,11 @@ type group struct {
 
 // Block/Allowlist items for blocklist-v2
 type list struct {
-	Name     string
-	Format   string
-	Source   string
-	CacheDir string `toml:"cache-dir"` // Where to store copies of remote blocklists for faster startup
+	Name     	 string
+	Format   	 string
+	Source   	 string
+	CacheDir 	 string `toml:"cache-dir"` // Where to store copies of remote blocklists for faster startup
+	AllowFailOnStart bool   `toml:"allow-fail-on-startup"` // Don't fail if the blocklist can't be loaded on startup, just print a warning
 }
 
 type router struct {

cmd/routedns/main.go

@@ -768,18 +768,28 @@ func newBlocklistDB(l list, rules []string) (rdns.BlocklistDB, error) {
 			return nil, fmt.Errorf("unsupported scheme '%s' in '%s'", loc.Scheme, l.Source)
 		}
 	}
+	var db rdns.BlocklistDB
 	switch l.Format {
 	case "regexp", "":
-		return rdns.NewRegexpDB(name, loader)
+		db = rdns.NewRegexpDB(name, loader)
 	case "domain":
-		return rdns.NewDomainDB(name, loader)
+		db = rdns.NewDomainDB(name, loader)
 	case "hosts":
-		return rdns.NewHostsDB(name, loader)
+		db = rdns.NewHostsDB(name, loader)
 	default:
 		return nil, fmt.Errorf("unsupported format '%s'", l.Format)
 	}
+	db, err = db.Reload()
+	if err != nil {
+		rdns.Log.WithError(err).Warn("failed to load list")
+		if !l.AllowFailOnStart {
+			return nil, fmt.Errorf("failed to load list on startup, set allow-fail-on-startup to skip: %w", err)
+		}
+	}
+	return db, nil
 }
 
+
 func newIPBlocklistDB(l list, locationDB string, rules []string) (rdns.IPBlocklistDB, error) {
 	loc, err := url.Parse(l.Source)
 	if err != nil {
@@ -805,15 +815,24 @@ func newIPBlocklistDB(l list, locationDB string, rules []string) (rdns.IPBlockli
 			return nil, fmt.Errorf("unsupported scheme '%s' in '%s'", loc.Scheme, l.Source)
 		}
 	}
-
+	var db rdns.IPBlocklistDB
 	switch l.Format {
-	case "cidr", "":
-		return rdns.NewCidrDB(name, loader)
-	case "location":
-		return rdns.NewGeoIPDB(name, loader, locationDB)
-	default:
-		return nil, fmt.Errorf("unsupported format '%s'", l.Format)
+		case "cidr", "":
+			db = rdns.NewCidrDB(name, loader)
+		case "location":
+			return rdns.NewGeoIPDB(name, loader, locationDB)
+		default:
+			return nil, fmt.Errorf("unsupported format '%s'", l.Format)
 	}
+	db, err = db.Reload()
+
+	if err != nil {
+                rdns.Log.WithError(err).Warn("failed to load list")
+                if !l.AllowFailOnStart {
+			return nil, fmt.Errorf("failed to load list on startup, set allow-fail-on-startup to skip: %w", err)
+		}
+        }
+        return db, nil
 }
 
 func printVersion() {

doc/configuration.md

@@ -1531,3 +1531,4 @@ protocol = "dot"
 ```
 
 Example config files: [bootstrap-resolver.toml](../cmd/routedns/example-config/bootstrap-resolver.toml), [use-case-6.toml](../cmd/routedns/example-config/use-case-6.toml)
+

go.sum

@@ -1,3 +1,4 @@
+
 github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
 github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/RackSec/srslog v0.0.0-20180709174129-a4725f04ec91 h1:vX+gnvBc56EbWYrmlhYbFYRaeikAke1GL84N4BEYOFE=
@@ -127,7 +128,7 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=