From 4e70f65eb9d1e28d5adb59ecd1b80d9a4e065f93 Mon Sep 17 00:00:00 2001
From: folbrich <frank.olbricht@gmail.com>
Date: Tue, 7 Apr 2020 18:52:46 -0600
Subject: [PATCH] Expand logging and documentation

---
 README.md                                         | 13 ++++++++++++-
 blocklistdb-domain_test.go                        | 10 ++++++++--
 blocklistloader-http.go                           |  3 +++
 blocklistloader-local.go                          |  3 +++
 cmd/routedns/example-config/blocklist-remote.toml |  2 +-
 5 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 4066668..f51435c 100644
--- a/README.md
+++ b/README.md
@@ -208,7 +208,7 @@ Some listeners, namely DoH and DoT, can be configured with certificates and can
 
 ## Blocklists
 
-Blocklists can be added to resolver-chains to prevent further processing and either return NXDOMAIN or a spoofed IP address. The blocklist group supports 2 types of blocklist formats:
+Blocklists can be added to resolver-chains to prevent further processing and either return NXDOMAIN or a spoofed IP address. The blocklist group supports 3 types of blocklist formats:
 
 - `regexp` - The entire query string is matched against a list of regular expressions and NXDOMAIN returned if a match is found.
 - `domain` - A list of domains with some wildcard capabilities. Also results in an NXDOMAIN. Entries in the list are matched as follows:
@@ -256,6 +256,17 @@ blocklist = [
 ]
 ```
 
+In addition to reading the blocklist rules from the configuration, routedns supports reading from the local filesystem and from remote servers via HTTP(S). Use the `source` property of the blocklist to provide the file location or URL. The `refresh` property can be used to specify a reload-period (in seconds). If no `refresh` period is given, the blocklist will only be loaded once at startup. The following example loads a regexp blocklist via HTTP once a day.
+
+```toml
+[groups.cloudflare-blocklist]
+type = "blocklist"
+resolvers = ["cloudflare-dot"]
+format    = "regexp"           # "domain", "hosts" or "regexp", defaults to "regexp"
+source    = "https://raw.githubusercontent.com/cbuijs/accomplist/master/deugniets/plain.black.regex.list"
+refresh   = 86400              # Time to refresh the blocklist from the file in seconds
+```
+
 ## Use-cases / Examples
 
 ### Use case 1: Use DNS-over-TLS for all queries locally
diff --git a/blocklistdb-domain_test.go b/blocklistdb-domain_test.go
index 743163f..7b2a8d4 100644
--- a/blocklistdb-domain_test.go
+++ b/blocklistdb-domain_test.go
@@ -13,7 +13,9 @@ func TestDomainDB(t *testing.T) {
 		".domain2.com.",   // exact match and subdomains
 		"x.domain2.com",   // above rule should take precendence
 		"*.domain3.com",   // subdomains only
-		"x.x.domain3.com", // more specific wildcard should take precedence
+		"x.x.domain3.com", // more general wildcard above should take precedence
+		"domain4.com",     // the more general rule below wins
+		".domain4.com",
 	)
 	require.NoError(t, err)
 
@@ -29,10 +31,14 @@ func TestDomainDB(t *testing.T) {
 		{"domain2.com.", true},
 		{"sub.domain2.com.", true},
 
-		// // wildcard (match only on subdomains)
+		// wildcard (match only on subdomains)
 		{"domain3.com.", false},
 		{"sub.domain3.com.", true},
 
+		// two rules for this, the generic one wins
+		{"domain4.com.", true},
+		{"sub.domain4.com.", true},
+
 		// not matching
 		{"unblocked.test.", false},
 		{"com.", false},
diff --git a/blocklistloader-http.go b/blocklistloader-http.go
index a6b3267..abe537e 100644
--- a/blocklistloader-http.go
+++ b/blocklistloader-http.go
@@ -22,6 +22,8 @@ func NewHTTPLoader(url string) *HTTPLoader {
 }
 
 func (l *HTTPLoader) Load() ([]string, error) {
+	log := Log.WithField("url", l.url)
+	log.Trace("loading blocklist")
 	ctx, cancel := context.WithTimeout(context.Background(), httpTimeout)
 	defer cancel()
 
@@ -45,5 +47,6 @@ func (l *HTTPLoader) Load() ([]string, error) {
 	for scanner.Scan() {
 		rules = append(rules, scanner.Text())
 	}
+	log.Trace("completed loading blocklist")
 	return rules, scanner.Err()
 }
diff --git a/blocklistloader-local.go b/blocklistloader-local.go
index d983fac..1b69bef 100644
--- a/blocklistloader-local.go
+++ b/blocklistloader-local.go
@@ -18,6 +18,8 @@ func NewFileLoader(filename string) *FileLoader {
 }
 
 func (l *FileLoader) Load() ([]string, error) {
+	log := Log.WithField("file", l.filename)
+	log.Trace("loading blocklist")
 	f, err := os.Open(l.filename)
 	if err != nil {
 		return nil, err
@@ -28,5 +30,6 @@ func (l *FileLoader) Load() ([]string, error) {
 	for scanner.Scan() {
 		rules = append(rules, scanner.Text())
 	}
+	log.Trace("completed loading blocklist")
 	return rules, scanner.Err()
 }
diff --git a/cmd/routedns/example-config/blocklist-remote.toml b/cmd/routedns/example-config/blocklist-remote.toml
index 2d71b82..aecf0d5 100644
--- a/cmd/routedns/example-config/blocklist-remote.toml
+++ b/cmd/routedns/example-config/blocklist-remote.toml
@@ -7,7 +7,7 @@ type = "blocklist"
 resolvers = ["cloudflare-dot"] # Anything that passes the filter is sent on to this resolver
 format    = "regexp"           # "domain", "hosts" or "regexp", defaults to "regexp"
 source    = "https://raw.githubusercontent.com/cbuijs/accomplist/master/deugniets/plain.black.regex.list"
-refresh   = 86400              # Time to refresh the blocklist from the file in seconds
+refresh   = 86400              # Time to refresh the blocklist from the URL in seconds
 
 [listeners.local-udp]
 address = ":53"