Add support for local files and remote blocklists (HTTP)

blocklist.go

@@ -4,6 +4,8 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"sync"
+	"time"
 
 	"github.com/miekg/dns"
 	"github.com/sirupsen/logrus"
@@ -14,13 +16,33 @@ import (
 type Blocklist struct {
 	resolver Resolver
 	db       BlocklistDB
+	loader   BlocklistLoader
+	mu       sync.Mutex
 }
 
 var _ Resolver = &Blocklist{}
 
-// NewBlocklist returns a new instance of a blocklist resolver.
-func NewBlocklist(resolver Resolver, db BlocklistDB) (*Blocklist, error) {
-	return &Blocklist{resolver: resolver, db: db}, nil
+// NewBlocklist returns a new instance of a blocklist resolver. If a non-nil loader is provided
+// the rules are loaded from it immediately into the DB. If refresh is >0, the rules are reloaded
+// periodically.
+func NewBlocklist(resolver Resolver, db BlocklistDB, l BlocklistLoader, refresh time.Duration) (*Blocklist, error) {
+	if l != nil {
+		rules, err := l.Load()
+		if err != nil {
+			return nil, fmt.Errorf("failed to retrieve rules: %w", err)
+		}
+		db, err = db.New(rules)
+		if err != nil {
+			return nil, fmt.Errorf("failed to load rules: %w", err)
+		}
+	}
+	blocklist := &Blocklist{resolver: resolver, db: db, loader: l}
+
+	// Start the refresh goroutine if we have a loader and a refresh period was given
+	if l != nil && refresh > 0 {
+		go blocklist.refreshLoop(refresh)
+	}
+	return blocklist, nil
 }
 
 // Resolve a DNS query by first checking the query against the provided matcher.
@@ -81,3 +103,24 @@ func (r *Blocklist) Resolve(q *dns.Msg, ci ClientInfo) (*dns.Msg, error) {
 func (r *Blocklist) String() string {
 	return fmt.Sprintf("Blocklist(%s)", r.db)
 }
+
+func (r *Blocklist) refreshLoop(refresh time.Duration) {
+	for {
+		time.Sleep(refresh)
+		Log.Debug("reloading blocklist")
+
+		rules, err := r.loader.Load()
+		if err != nil {
+			Log.WithError(err).Error("failed to retrieve rules")
+			continue
+		}
+		db, err := r.db.New(rules)
+		if err != nil {
+			Log.WithError(err).Error("failed to load rules")
+			continue
+		}
+		r.mu.Lock()
+		r.db = db
+		r.mu.Unlock()
+	}
+}

blocklist_test.go

@@ -15,7 +15,7 @@ func TestBlocklistRegexp(t *testing.T) {
 	m, err := NewRegexpDB(`(^|\.)block\.test`, `(^|\.)evil\.test`)
 	require.NoError(t, err)
 
-	b, err := NewBlocklist(r, m)
+	b, err := NewBlocklist(r, m, nil, 0)
 	require.NoError(t, err)
 
 	// First query a domain not blocked. Should be passed through to the resolver

blocklistdb-domain.go

@@ -22,15 +22,15 @@ type node map[string]node
 var _ BlocklistDB = &DomainDB{}
 
 // NewDomainDB returns a new instance of a matcher for a list of regular expressions.
-func NewDomainDB(filter ...string) (*DomainDB, error) {
+func NewDomainDB(rules ...string) (*DomainDB, error) {
 	root := make(node)
-	for _, s := range filter {
+	for _, r := range rules {
 		// Strip trailing . in case the list has FQDN names with . suffixes.
-		s = strings.TrimSuffix(s, ".")
+		r = strings.TrimSuffix(r, ".")
 
 		// Break up the domain into its parts and iterare backwards over them, building
 		// a graph of maps
-		parts := strings.Split(s, ".")
+		parts := strings.Split(r, ".")
 		n := root
 		for i := len(parts) - 1; i >= 0; i-- {
 			part := parts[i]
@@ -51,6 +51,10 @@ func NewDomainDB(filter ...string) (*DomainDB, error) {
 	return &DomainDB{root: root}, nil
 }
 
+func (m *DomainDB) New(rules []string) (BlocklistDB, error) {
+	return NewDomainDB(rules...)
+}
+
 func (m *DomainDB) Match(q dns.Question) (net.IP, bool) {
 	s := strings.TrimSuffix(q.Name, ".")
 	parts := strings.Split(s, ".")

blocklistdb-hosts.go

@@ -22,10 +22,10 @@ type ipRecords struct {
 var _ BlocklistDB = &HostsDB{}
 
 // NewHostsDB returns a new instance of a matcher for a list of regular expressions.
-func NewHostsDB(items ...string) (*HostsDB, error) {
+func NewHostsDB(rules ...string) (*HostsDB, error) {
 	filters := make(map[string]ipRecords)
-	for _, s := range items {
-		fields := strings.Fields(s)
+	for _, r := range rules {
+		fields := strings.Fields(r)
 		if len(fields) == 0 {
 			continue
 		}
@@ -56,6 +56,10 @@ func NewHostsDB(items ...string) (*HostsDB, error) {
 	return &HostsDB{filters}, nil
 }
 
+func (m *HostsDB) New(rules []string) (BlocklistDB, error) {
+	return NewHostsDB(rules...)
+}
+
 func (m *HostsDB) Match(q dns.Question) (net.IP, bool) {
 	ips, ok := m.filters[strings.TrimSuffix(q.Name, ".")]
 	if q.Qtype == dns.TypeA {

blocklistdb-regexp.go

@@ -15,10 +15,10 @@ type RegexpDB struct {
 var _ BlocklistDB = &RegexpDB{}
 
 // NewRegexpDB returns a new instance of a matcher for a list of regular expressions.
-func NewRegexpDB(items ...string) (*RegexpDB, error) {
+func NewRegexpDB(rules ...string) (*RegexpDB, error) {
 	var filters []*regexp.Regexp
-	for _, s := range items {
-		re, err := regexp.Compile(s)
+	for _, r := range rules {
+		re, err := regexp.Compile(r)
 		if err != nil {
 			return nil, err
 		}
@@ -28,6 +28,10 @@ func NewRegexpDB(items ...string) (*RegexpDB, error) {
 	return &RegexpDB{filters}, nil
 }
 
+func (m *RegexpDB) New(rules []string) (BlocklistDB, error) {
+	return NewRegexpDB(rules...)
+}
+
 func (m *RegexpDB) Match(q dns.Question) (net.IP, bool) {
 	for _, filter := range m.filters {
 		if filter.MatchString(q.Name) {

blocklistdb.go

@@ -8,6 +8,11 @@ import (
 )
 
 type BlocklistDB interface {
+	// New initializes a new instance of the same database but with
+	// the rules passed into it. Used to load new rules during an
+	// ruleset refresh.
+	New(rules []string) (BlocklistDB, error)
+
 	// Returns true if the question matches a record. If the IP is not nil,
 	// respond with the given IP. NXDOMAIN otherwise.
 	Match(q dns.Question) (net.IP, bool)

blocklistloader-http.go

@@ -0,0 +1,49 @@
+package rdns
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"net/http"
+	"time"
+)
+
+// HTTPLoader reads blocklist rules from a server via HTTP(S).
+type HTTPLoader struct {
+	url string
+}
+
+var _ BlocklistLoader = &HTTPLoader{}
+
+const httpTimeout = 30 * time.Minute
+
+func NewHTTPLoader(url string) *HTTPLoader {
+	return &HTTPLoader{url}
+}
+
+func (l *HTTPLoader) Load() ([]string, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), httpTimeout)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, "GET", l.url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		return nil, fmt.Errorf("got unexpected status code %d from %s", resp.StatusCode, l.url)
+	}
+
+	var rules []string
+	scanner := bufio.NewScanner(resp.Body)
+	for scanner.Scan() {
+		rules = append(rules, scanner.Text())
+	}
+	return rules, scanner.Err()
+}

blocklistloader-local.go

@@ -0,0 +1,32 @@
+package rdns
+
+import (
+	"bufio"
+	"os"
+)
+
+// FileLoader reads blocklist rules from a local file. Used to refresh blocklists
+// from a file on the local machine.
+type FileLoader struct {
+	filename string
+}
+
+var _ BlocklistLoader = &FileLoader{}
+
+func NewFileLoader(filename string) *FileLoader {
+	return &FileLoader{filename}
+}
+
+func (l *FileLoader) Load() ([]string, error) {
+	f, err := os.Open(l.filename)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	var rules []string
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		rules = append(rules, scanner.Text())
+	}
+	return rules, scanner.Err()
+}

blocklistloader-static.go

@@ -0,0 +1,17 @@
+package rdns
+
+// StaticLoader holds a fixed ruleset in memory. It's used for loading fixed
+// blocklists from configuration that doesn't get refreshed.
+type StaticLoader struct {
+	rules []string
+}
+
+var _ BlocklistLoader = &StaticLoader{}
+
+func NewStaticLoader(rules []string) *StaticLoader {
+	return &StaticLoader{rules}
+}
+
+func (l *StaticLoader) Load() ([]string, error) {
+	return l.rules, nil
+}

blocklistloader.go

@@ -0,0 +1,6 @@
+package rdns
+
+type BlocklistLoader interface {
+	// Returns a list of rules that can then be stored into a blocklist DB.
+	Load() ([]string, error)
+}

cmd/routedns/config.go

@@ -45,8 +45,10 @@ type doh struct {
 type group struct {
 	Resolvers []string
 	Type      string
-	Blocklist []string                // only used by "blocklist" type
+	Blocklist []string                // Blocklist rules, only used by "blocklist" type
 	Format    string                  // Blocklist input format: "regex", "domain", or "hosts"
+	Source    string                  // Location of external blocklist, can be a local path or remote URL
+	Refresh   int                     // Blocklist refresh when using an external source, in seconds
 	Replace   []rdns.ReplaceOperation // only used by "replace" type
 	GCPeriod  int                     `toml:"gc-period"` // Time-period (seconds) used to expire cached items in the "cache" type
 }

cmd/routedns/example-config/blocklist-local.toml

@@ -0,0 +1,20 @@
+[resolvers.cloudflare-dot]
+address = "1.1.1.1:853"
+protocol = "dot"
+
+[groups.cloudflare-blocklist]
+type = "blocklist"
+resolvers = ["cloudflare-dot"] # Anything that passes the filter is sent on to this resolver
+format    = "domain"           # "domain", "hosts" or "regexp", defaults to "regexp"
+source    = "./example-config/domains.txt"    # Location of the local blocklist file.
+refresh   = 86400              # Time to refresh the blocklist from the file in seconds
+
+[listeners.local-udp]
+address = ":1153"
+protocol = "udp"
+resolver = "cloudflare-blocklist"
+
+[listeners.local-tcp]
+address = ":1153"
+protocol = "tcp"
+resolver = "cloudflare-blocklist"

cmd/routedns/example-config/blocklist-remote.toml

@@ -0,0 +1,20 @@
+[resolvers.cloudflare-dot]
+address = "1.1.1.1:853"
+protocol = "dot"
+
+[groups.cloudflare-blocklist]
+type = "blocklist"
+resolvers = ["cloudflare-dot"] # Anything that passes the filter is sent on to this resolver
+format    = "regexp"           # "domain", "hosts" or "regexp", defaults to "regexp"
+source    = "https://raw.githubusercontent.com/cbuijs/accomplist/master/deugniets/plain.black.regex.list"
+refresh   = 86400              # Time to refresh the blocklist from the file in seconds
+
+[listeners.local-udp]
+address = ":1153"
+protocol = "udp"
+resolver = "cloudflare-blocklist"
+
+[listeners.local-tcp]
+address = ":1153"
+protocol = "tcp"
+resolver = "cloudflare-blocklist"

cmd/routedns/example-config/domains.txt

@@ -0,0 +1,5 @@
+# Example list of domains on a blocklist
+
+.evil.com
+blocked.com
+facebook.com

cmd/routedns/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"net/url"
 	"os"
 	"time"
 
@@ -131,6 +132,24 @@ func start(opt options, args []string) error {
 			if len(gr) != 1 {
 				return fmt.Errorf("type blocklist only supports one resolver in '%s'", id)
 			}
+			if len(g.Blocklist) > 0 && g.Source != "" {
+				return fmt.Errorf("type static blocklist can't be used with 'source' in '%s'", id)
+			}
+			var loader rdns.BlocklistLoader
+			if g.Source != "" {
+				loc, err := url.Parse(g.Source)
+				if err != nil {
+					return err
+				}
+				switch loc.Scheme {
+				case "http", "https":
+					loader = rdns.NewHTTPLoader(g.Source)
+				case "":
+					loader = rdns.NewFileLoader(g.Source)
+				default:
+					return fmt.Errorf("unsupported scheme '%s' in '%s'", loc.Scheme, g.Source)
+				}
+			}
 			var db rdns.BlocklistDB
 			switch g.Format {
 			case "regexp", "":
@@ -151,7 +170,7 @@ func start(opt options, args []string) error {
 			default:
 				return fmt.Errorf("unsupported blocklist format '%s'", g.Format)
 			}
-			resolvers[id], err = rdns.NewBlocklist(gr[0], db)
+			resolvers[id], err = rdns.NewBlocklist(gr[0], db, loader, time.Duration(g.Refresh)*time.Second)
 			if err != nil {
 				return err
 			}