Expand logging and documentation

README.md

@@ -208,7 +208,7 @@ Some listeners, namely DoH and DoT, can be configured with certificates and can
 
 ## Blocklists
 
-Blocklists can be added to resolver-chains to prevent further processing and either return NXDOMAIN or a spoofed IP address. The blocklist group supports 2 types of blocklist formats:
+Blocklists can be added to resolver-chains to prevent further processing and either return NXDOMAIN or a spoofed IP address. The blocklist group supports 3 types of blocklist formats:
 
 - `regexp` - The entire query string is matched against a list of regular expressions and NXDOMAIN returned if a match is found.
 - `domain` - A list of domains with some wildcard capabilities. Also results in an NXDOMAIN. Entries in the list are matched as follows:
@@ -256,6 +256,17 @@ blocklist = [
 ]
 ```
 
+In addition to reading the blocklist rules from the configuration, routedns supports reading from the local filesystem and from remote servers via HTTP(S). Use the `source` property of the blocklist to provide the file location or URL. The `refresh` property can be used to specify a reload-period (in seconds). If no `refresh` period is given, the blocklist will only be loaded once at startup. The following example loads a regexp blocklist via HTTP once a day.
+
+```toml
+[groups.cloudflare-blocklist]
+type = "blocklist"
+resolvers = ["cloudflare-dot"]
+format    = "regexp"           # "domain", "hosts" or "regexp", defaults to "regexp"
+source    = "https://raw.githubusercontent.com/cbuijs/accomplist/master/deugniets/plain.black.regex.list"
+refresh   = 86400              # Time to refresh the blocklist from the file in seconds
+```
+
 ## Use-cases / Examples
 
 ### Use case 1: Use DNS-over-TLS for all queries locally

blocklistdb-domain_test.go

@@ -13,7 +13,9 @@ func TestDomainDB(t *testing.T) {
 		".domain2.com.",   // exact match and subdomains
 		"x.domain2.com",   // above rule should take precendence
 		"*.domain3.com",   // subdomains only
-		"x.x.domain3.com", // more specific wildcard should take precedence
+		"x.x.domain3.com", // more general wildcard above should take precedence
+		"domain4.com",     // the more general rule below wins
+		".domain4.com",
 	)
 	require.NoError(t, err)
 
@@ -29,10 +31,14 @@ func TestDomainDB(t *testing.T) {
 		{"domain2.com.", true},
 		{"sub.domain2.com.", true},
 
-		// // wildcard (match only on subdomains)
+		// wildcard (match only on subdomains)
 		{"domain3.com.", false},
 		{"sub.domain3.com.", true},
 
+		// two rules for this, the generic one wins
+		{"domain4.com.", true},
+		{"sub.domain4.com.", true},
+
 		// not matching
 		{"unblocked.test.", false},
 		{"com.", false},

blocklistloader-http.go

@@ -22,6 +22,8 @@ func NewHTTPLoader(url string) *HTTPLoader {
 }
 
 func (l *HTTPLoader) Load() ([]string, error) {
+	log := Log.WithField("url", l.url)
+	log.Trace("loading blocklist")
 	ctx, cancel := context.WithTimeout(context.Background(), httpTimeout)
 	defer cancel()
 
@@ -45,5 +47,6 @@ func (l *HTTPLoader) Load() ([]string, error) {
 	for scanner.Scan() {
 		rules = append(rules, scanner.Text())
 	}
+	log.Trace("completed loading blocklist")
 	return rules, scanner.Err()
 }

blocklistloader-local.go

@@ -18,6 +18,8 @@ func NewFileLoader(filename string) *FileLoader {
 }
 
 func (l *FileLoader) Load() ([]string, error) {
+	log := Log.WithField("file", l.filename)
+	log.Trace("loading blocklist")
 	f, err := os.Open(l.filename)
 	if err != nil {
 		return nil, err
@@ -28,5 +30,6 @@ func (l *FileLoader) Load() ([]string, error) {
 	for scanner.Scan() {
 		rules = append(rules, scanner.Text())
 	}
+	log.Trace("completed loading blocklist")
 	return rules, scanner.Err()
 }

cmd/routedns/example-config/blocklist-remote.toml

@@ -7,7 +7,7 @@ type = "blocklist"
 resolvers = ["cloudflare-dot"] # Anything that passes the filter is sent on to this resolver
 format    = "regexp"           # "domain", "hosts" or "regexp", defaults to "regexp"
 source    = "https://raw.githubusercontent.com/cbuijs/accomplist/master/deugniets/plain.black.regex.list"
-refresh   = 86400              # Time to refresh the blocklist from the file in seconds
+refresh   = 86400              # Time to refresh the blocklist from the URL in seconds
 
 [listeners.local-udp]
 address = ":53"