mirror/goaway
1
0
Fork 0
mirror of https://github.com/pommee/goaway.git synced 2026-05-24 03:49:21 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat: added ability to update lists

Browse Source
This commit is contained in:
pommee
2025-05-01 09:15:36 +02:00
parent f2ba2b09bf
commit 6ac1f8cbaa
7 changed files with 262 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+10 -1
View File
@@ -16,7 +16,16 @@ require (
golang.org/x/crypto v0.37.0
)
require github.com/gin-contrib/gzip v1.2.3
require (
github.com/gin-contrib/gzip v1.2.3
github.com/stretchr/testify v1.10.0
golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0
)
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
)
require (
github.com/bytedance/sonic v1.13.2 // indirect
go.sum
+2
View File
@@ -111,6 +111,8 @@ golang.org/x/arch v0.16.0 h1:foMtLTdyOmIniqWCHjY6+JxuC54XP1fDwx4N0ASyW+U=
golang.org/x/arch v0.16.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE=
golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM=
golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8=
golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
internal/api/api.go
+2
View File
@@ -159,6 +159,8 @@ func (api *API) setupAuthorizedRoutes(devmode bool) {
api.routes.GET("/topClients", api.getTopClients)
api.routes.GET("/lists", api.getLists)
api.routes.GET("/addList", api.addList)
api.routes.GET("/fetchUpdatedList", api.fetchUpdatedList)
api.routes.GET("/runUpdateList", api.runUpdateList)
api.routes.GET("/getDomainsForList", api.getDomainsForList)
api.routes.GET("/runUpdate", api.runUpdate)
api.routes.GET("/pause", api.getBlocking)
internal/api/routes.go
+83
View File
@@ -770,6 +770,89 @@ func (api *API) addList(c *gin.Context) {
c.JSON(http.StatusOK, nil)
}
func (api *API) fetchUpdatedList(c *gin.Context) {
name := c.Query("name")
url := c.Query("url")
if api.DnsServer.Blacklist.BlocklistURL[name] == "" {
c.JSON(http.StatusBadRequest, gin.H{"error": "List does not exist"})
return
}
if name == "" || url == "" {
c.JSON(http.StatusBadRequest, gin.H{"error": "Missing 'name' or 'url' query parameter"})
return
}
remoteDomains, remoteChecksum, err := api.DnsServer.Blacklist.FetchRemoteHostsList(url, name)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err})
return
}
dbDomains, dbChecksum, err := api.DnsServer.Blacklist.FetchDBHostsList(url, name)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err})
return
}
if remoteChecksum == dbChecksum {
c.JSON(http.StatusOK, gin.H{"updateAvailable": false, "message": "No list updates available"})
return
}
diff := func(a, b []string) []string {
mb := make(map[string]struct{}, len(b))
for _, x := range b {
mb[x] = struct{}{}
}
diff := make([]string, 0)
for _, x := range a {
if _, found := mb[x]; !found {
diff = append(diff, x)
}
}
return diff
}
c.JSON(http.StatusOK, gin.H{
"updateAvailable": true,
"remoteChecksum": remoteChecksum,
"dbChecksum": dbChecksum,
"diffAdded": diff(remoteDomains, dbDomains),
"diffRemoved": diff(dbDomains, remoteDomains),
})
}
func (api *API) runUpdateList(c *gin.Context) {
name := c.Query("name")
url := c.Query("url")
if api.DnsServer.Blacklist.BlocklistURL[name] == "" {
c.JSON(http.StatusBadRequest, gin.H{"error": "List does not exist"})
return
}
if name == "" || url == "" {
c.JSON(http.StatusBadRequest, gin.H{"error": "Missing 'name' or 'url' query parameter"})
return
}
err := api.DnsServer.Blacklist.RemoveSourceAndDomains(name)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err})
return
}
err = api.DnsServer.Blacklist.FetchAndLoadHosts(url, name)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err})
return
}
api.DnsServer.Blacklist.PopulateBlocklistCache()
c.Status(http.StatusOK)
}
func (api *API) removeList(c *gin.Context) {
name := c.Query("name")
internal/blacklist/blacklist.go
+52 -13
View File
@@ -2,14 +2,19 @@ package blacklist
import (
"bufio"
"crypto/sha256"
"database/sql"
"encoding/hex"
"errors"
"fmt"
"goaway/internal/logging"
"io"
"net/http"
"sort"
"strings"
"time"
"golang.org/x/exp/slices"
)
var log = logging.GetLogger()
@@ -106,6 +111,38 @@ func (b *Blacklist) GetBlocklistUrls() (map[string]string, error) {
return blocklistURL, nil
}
func (b *Blacklist) FetchRemoteHostsList(url, name string) ([]string, string, error) {
resp, err := http.Get(url)
if err != nil {
return nil, "", fmt.Errorf("failed to fetch hosts file from %s: %w", url, err)
}
defer resp.Body.Close()
domains, err := b.ExtractDomains(resp.Body)
if err != nil {
return nil, "", fmt.Errorf("failed to extract domains from %s: %w", url, err)
}
return domains, calculateDomainsChecksum(domains), nil
}
func (b *Blacklist) FetchDBHostsList(url, name string) ([]string, string, error) {
domains, err := b.GetDomainsForList(name)
if err != nil {
return nil, "", fmt.Errorf("could not fetch domains from database")
}
return domains, calculateDomainsChecksum(domains), nil
}
func calculateDomainsChecksum(domains []string) string {
sort.Strings(domains)
data := strings.Join(domains, "\n")
hash := sha256.Sum256([]byte(data))
return hex.EncodeToString(hash[:])
}
func (b *Blacklist) FetchAndLoadHosts(url, name string) error {
resp, err := http.Get(url)
if err != nil {
@@ -115,7 +152,7 @@ func (b *Blacklist) FetchAndLoadHosts(url, name string) error {
_ = Body.Close()
}(resp.Body)
domains, err := b.extractDomains(resp.Body)
domains, err := b.ExtractDomains(resp.Body)
if err != nil {
return fmt.Errorf("failed to extract domains from %s: %w", url, err)
}
@@ -130,36 +167,38 @@ func (b *Blacklist) FetchAndLoadHosts(url, name string) error {
return nil
}
func (b *Blacklist) extractDomains(body io.Reader) ([]string, error) {
var domains []string
func (b *Blacklist) ExtractDomains(body io.Reader) ([]string, error) {
scanner := bufio.NewScanner(body)
var domains []string
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if len(line) == 0 || line[0] == '#' {
fields := strings.Fields(scanner.Text())
if len(fields) == 0 || strings.HasPrefix(fields[0], "#") {
continue
}
parts := strings.Fields(line)
if len(parts) > 1 {
switch parts[1] {
case "localhost", "localhost.localdomain", "broadcasthost", "local":
domain := fields[0]
if fields[0] == "0.0.0.0" || fields[0] == "127.0.0.1" && len(fields) > 1 {
domain = fields[1]
switch domain {
case "localhost", "localhost.localdomain", "broadcasthost", "local", "0.0.0.0":
continue
}
domains = append(domains, parts[1:2]...)
} else if domain == "0.0.0.0" || domain == "127.0.0.1" {
continue
}
domains = append(domains, parts[0])
domains = append(domains, domain)
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("error reading hosts file: %w", err)
}
if len(domains) == 0 {
return nil, errors.New("zero results when parsing")
}
return domains, nil
return slices.Compact(domains), nil
}
func (b *Blacklist) AddDomain(domain string) error {
internal/database/request.go
+1 -1
View File
@@ -387,7 +387,7 @@ func DeleteRequestLogsTimebased(db *sql.DB, requestThreshold, maxRetries int, re
result, err := db.Exec(query)
if err != nil {
if err.Error() == "database is locked" {
log.Debug("Database is locked; retrying (%d/%d)", retryCount+1, maxRetries)
log.Warning("Database is locked; retrying (%d/%d)", retryCount+1, maxRetries)
time.Sleep(retryDelay)
continue
}
website/src/app/lists/details.tsx
+112 -1
View File
@@ -22,6 +22,11 @@ import BlockedDomainsList from "./blockedDomains";
export function CardDetails(listEntry: ListEntry) {
const [dialogOpen, setDialogOpen] = useState(false);
const [updateDiff, setUpdateDiff] = useState({
diffAdded: [],
diffRemoved: []
});
const [showDiff, setShowDiff] = useState(false);
const toggleBlocklist = async () => {
const [code, response] = await GetRequest(
@@ -36,6 +41,59 @@ export function CardDetails(listEntry: ListEntry) {
}
};
const checkForUpdates = async () => {
try {
const [code, response] = await GetRequest(
`fetchUpdatedList?name=${encodeURIComponent(listEntry.name)}&url=${
listEntry.url || ""
}`
);
if (code === 200) {
if (response.updateAvailable) {
setUpdateDiff({
diffAdded: response.diffAdded || [],
diffRemoved: response.diffRemoved || []
});
setShowDiff(true);
} else {
toast.info("No updates available");
setShowDiff(false);
}
} else {
toast.error(response.error);
setShowDiff(false);
}
} catch (error) {
console.log(error);
toast.error("Error checking for updates");
setShowDiff(false);
}
};
const runUpdateList = async () => {
try {
const [code, response] = await GetRequest(
`runUpdateList?name=${encodeURIComponent(listEntry.name)}&url=${
listEntry.url || ""
}`
);
if (code === 200) {
setDialogOpen(false);
setShowDiff(false);
toast.info(`Updated ${listEntry.name}`);
} else {
toast.error(response.error);
setShowDiff(false);
}
} catch (error) {
console.log(error);
toast.error("Error checking for updates");
setShowDiff(false);
}
};
return (
<Dialog open={dialogOpen} onOpenChange={setDialogOpen}>
<DialogTrigger asChild>
@@ -98,11 +156,12 @@ export function CardDetails(listEntry: ListEntry) {
{listEntry.name !== "Custom" && (
<>
<Button
onClick={checkForUpdates}
variant="outline"
className="bg-blue-600 border-none hover:bg-blue-500 text-white flex-1 text-sm"
>
<ArrowsClockwise className="mr-1" size={16} />
Update [WIP]
Update
</Button>
<Button
variant="outline"
@@ -117,6 +176,58 @@ export function CardDetails(listEntry: ListEntry) {
{listEntry.name === "Custom" && (
<BlockedDomainsList listName={listEntry.name} />
)}
{showDiff && (
<div className="mt-4 p-4 bg-zinc-800 rounded-lg border border-zinc-700">
<h3 className="font-bold mb-2">
Update for {listEntry.name} found
</h3>
<Separator className="bg-stone-700 mb-1" />
{updateDiff.diffAdded.length > 0 && (
<>
<div className="mb-3">
<h4 className="text-green-400 mb-1">
New Domains: {updateDiff.diffAdded.length}{" "}
</h4>
<div className="max-h-24 overflow-y-auto text-xs flex gap-1 flex-wrap">
{updateDiff.diffAdded.map((item, i) => (
<div
key={`added-${i}`}
className="text-green-300 bg-stone-900 p-1 rounded-sm"
>
{item}
</div>
))}
</div>
</div>
<Separator className="bg-stone-600 mb-2" />
</>
)}
{updateDiff.diffRemoved.length > 0 && (
<div>
<h4 className="text-red-400 mb-1">
Deleted Domains: {updateDiff.diffRemoved.length}
</h4>
<div className="max-h-24 overflow-y-auto text-xs flex gap-1 flex-wrap">
{updateDiff.diffRemoved.map((item, i) => (
<div
key={`removed-${i}`}
className="text-red-300 bg-stone-900 p-1 rounded-sm"
>
{item}
</div>
))}
</div>
</div>
)}
<Button
className="mt-6 bg-green-700 hover:bg-green-600 cursor-pointer text-white w-full"
onClick={runUpdateList}
>
Accept changes
</Button>
</div>
)}
</DialogContent>
</Dialog>
);