refactor: move some io, http and string utils to separate repo

This commit is contained in:
yusing
2025-09-27 12:46:41 +08:00
parent acecd827d6
commit 22bcf1201b
115 changed files with 125 additions and 3815 deletions

View File

@@ -12,9 +12,9 @@ import (
"github.com/yusing/godoxy/internal/metrics/systeminfo"
httpServer "github.com/yusing/godoxy/internal/net/gphttp/server"
"github.com/yusing/godoxy/internal/task"
"github.com/yusing/godoxy/internal/utils/strutils"
"github.com/yusing/godoxy/pkg"
socketproxy "github.com/yusing/godoxy/socketproxy/pkg"
strutils "github.com/yusing/goutils/strings"
)
func main() {

View File

@@ -19,9 +19,8 @@ require (
github.com/rs/zerolog v1.34.0
github.com/stretchr/testify v1.11.1
github.com/yusing/godoxy v0.18.6
github.com/yusing/godoxy/internal/utils v0.0.0
github.com/yusing/godoxy/socketproxy v0.0.0-00010101000000-000000000000
github.com/yusing/goutils v0.0.0-20250922091446-1c6a11717d72
github.com/yusing/goutils v0.2.1
)
require (
@@ -89,7 +88,8 @@ require (
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.3.0 // indirect
github.com/vincent-petithory/dataurl v1.0.0 // indirect
github.com/yusing/ds v0.1.0 // indirect
github.com/yusing/ds v0.2.0 // indirect
github.com/yusing/godoxy/internal/utils v0.1.0 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect

View File

@@ -206,10 +206,10 @@ github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2W
github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI=
github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yusing/ds v0.1.0 h1:aiZs7jPMN3MEChUsddMYjpZFHhhAmkxrwRyIUnGy5AU=
github.com/yusing/ds v0.1.0/go.mod h1:KC785+mtt+Bau0LLR+slExDaUjeiqLT1k9Or6Rpryh4=
github.com/yusing/goutils v0.0.0-20250922091446-1c6a11717d72 h1:NHYq8ZqoLSCJYzZaIOn8mJ28/Ac7GEFgeZgcfBoamWE=
github.com/yusing/goutils v0.0.0-20250922091446-1c6a11717d72/go.mod h1:XROGErdAT8UxsbLQbpejGhrTpknWEEgy+9HVjq7ULBI=
github.com/yusing/ds v0.2.0 h1:lPhDU5eA2uvquVrBrzLCrQXRJJgSXlUYA53TbuK2sQY=
github.com/yusing/ds v0.2.0/go.mod h1:XhKV4l7cZwBbbl7lRzNC9zX27zvCM0frIwiuD40ULRk=
github.com/yusing/goutils v0.2.1 h1:KjoCrNO0otthaPCZPfQY+5GKsqs5+J77CxP+TNHYa/Y=
github.com/yusing/goutils v0.2.1/go.mod h1:v6RZsMRdzcts4udSg0vqUIFvaD0OaUMPTwYJZ4XnQYo=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=

View File

@@ -6,8 +6,7 @@ import (
"net/http"
"github.com/gorilla/websocket"
"github.com/yusing/godoxy/internal/net/gphttp/reverseproxy"
nettypes "github.com/yusing/godoxy/internal/net/types"
"github.com/yusing/goutils/http/reverseproxy"
)
func (cfg *AgentConfig) Do(ctx context.Context, method, endpoint string, body io.Reader) (*http.Response, error) {
@@ -74,7 +73,7 @@ func (cfg *AgentConfig) Websocket(ctx context.Context, endpoint string) (*websoc
// It will create a new request with the same context, method, and body, but with the agent host and scheme, and the endpoint
// If the request has a query, it will be added to the proxy request's URL
func (cfg *AgentConfig) ReverseProxy(w http.ResponseWriter, req *http.Request, endpoint string) {
rp := reverseproxy.NewReverseProxy("agent", nettypes.NewURL(AgentURL), cfg.Transport())
rp := reverseproxy.NewReverseProxy("agent", AgentURL, cfg.Transport())
req.URL.Host = AgentHost
req.URL.Scheme = "https"
req.URL.Path = endpoint

View File

@@ -6,7 +6,7 @@ import (
"io"
"path/filepath"
"github.com/yusing/godoxy/internal/utils/strutils"
strutils "github.com/yusing/goutils/strings"
)
const AgentCertsBasePath = "certs"

View File

@@ -10,7 +10,7 @@ import (
"time"
"github.com/rs/zerolog/log"
"github.com/yusing/godoxy/internal/utils/strutils"
strutils "github.com/yusing/goutils/strings"
)
func initProfiling() {

15
go.mod
View File

@@ -17,12 +17,14 @@ require (
github.com/coreos/go-oidc/v3 v3.15.0 // oidc authentication
github.com/docker/docker v28.4.0+incompatible // docker daemon
github.com/fsnotify/fsnotify v1.9.0 // file watcher
github.com/gin-gonic/gin v1.11.0 // api server
github.com/go-acme/lego/v4 v4.26.0 // acme client
github.com/go-playground/validator/v10 v10.27.0 // validator
github.com/gobwas/glob v0.2.3 // glob matcher for route rules
github.com/gorilla/websocket v1.5.3 // websocket for API and agent
github.com/gotify/server/v2 v2.7.3 // reference the Message struct for json response
github.com/lithammer/fuzzysearch v1.1.8 // fuzzy search for searching icons and filtering metrics
github.com/pires/go-proxyproto v0.8.1 // proxy protocol support
github.com/puzpuzpuz/xsync/v4 v4.2.0 // lock free map for concurrent operations
github.com/rs/zerolog v1.34.0 // logging
github.com/shirou/gopsutil/v4 v4.25.8 // system info metrics
@@ -40,13 +42,15 @@ require (
github.com/golang-jwt/jwt/v5 v5.3.0
github.com/luthermonson/go-proxmox v0.2.3
github.com/oschwald/maxminddb-golang v1.13.1
github.com/quic-go/quic-go v0.54.1
github.com/quic-go/quic-go v0.54.1 // http3 support
github.com/samber/slog-zerolog/v2 v2.7.3
github.com/spf13/afero v1.15.0
github.com/stretchr/testify v1.11.1
github.com/yusing/ds v0.2.0
github.com/yusing/godoxy/agent v0.0.0-20250926130035-55c1c918ba95
github.com/yusing/godoxy/internal/dnsproviders v0.0.0-20250926130035-55c1c918ba95
github.com/yusing/godoxy/internal/utils v0.0.0
github.com/yusing/godoxy/internal/utils v0.1.0
github.com/yusing/goutils v0.2.1
)
require (
@@ -210,13 +214,6 @@ require (
gopkg.in/yaml.v3 v3.0.1 // indirect
)
require (
github.com/gin-gonic/gin v1.11.0
github.com/pires/go-proxyproto v0.8.1
github.com/yusing/ds v0.1.0
github.com/yusing/goutils v0.0.0-20250922091446-1c6a11717d72
)
require (
github.com/akamai/AkamaiOPEN-edgegrid-golang/v11 v11.1.0 // indirect
github.com/aziontech/azionapi-go-sdk v0.143.0 // indirect

8
go.sum
View File

@@ -1646,10 +1646,10 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yusing/ds v0.1.0 h1:aiZs7jPMN3MEChUsddMYjpZFHhhAmkxrwRyIUnGy5AU=
github.com/yusing/ds v0.1.0/go.mod h1:KC785+mtt+Bau0LLR+slExDaUjeiqLT1k9Or6Rpryh4=
github.com/yusing/goutils v0.0.0-20250922091446-1c6a11717d72 h1:NHYq8ZqoLSCJYzZaIOn8mJ28/Ac7GEFgeZgcfBoamWE=
github.com/yusing/goutils v0.0.0-20250922091446-1c6a11717d72/go.mod h1:XROGErdAT8UxsbLQbpejGhrTpknWEEgy+9HVjq7ULBI=
github.com/yusing/ds v0.2.0 h1:lPhDU5eA2uvquVrBrzLCrQXRJJgSXlUYA53TbuK2sQY=
github.com/yusing/ds v0.2.0/go.mod h1:XhKV4l7cZwBbbl7lRzNC9zX27zvCM0frIwiuD40ULRk=
github.com/yusing/goutils v0.2.1 h1:KjoCrNO0otthaPCZPfQY+5GKsqs5+J77CxP+TNHYa/Y=
github.com/yusing/goutils v0.2.1/go.mod h1:v6RZsMRdzcts4udSg0vqUIFvaD0OaUMPTwYJZ4XnQYo=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=

View File

@@ -6,8 +6,8 @@ import (
"github.com/gin-gonic/gin"
"github.com/yusing/godoxy/agent/pkg/agent"
"github.com/yusing/godoxy/internal/net/gphttp/httpheaders"
"github.com/yusing/godoxy/internal/net/gphttp/websocket"
"github.com/yusing/goutils/http/httpheaders"
)
// @x-id "list"

View File

@@ -7,7 +7,7 @@ import (
dockerSystem "github.com/docker/docker/api/types/system"
"github.com/gin-gonic/gin"
"github.com/yusing/godoxy/internal/gperr"
"github.com/yusing/godoxy/internal/utils/strutils"
strutils "github.com/yusing/goutils/strings"
)
type containerStats struct {

View File

@@ -11,8 +11,8 @@ import (
config "github.com/yusing/godoxy/internal/config/types"
"github.com/yusing/godoxy/internal/docker"
"github.com/yusing/godoxy/internal/gperr"
"github.com/yusing/godoxy/internal/net/gphttp/httpheaders"
"github.com/yusing/godoxy/internal/net/gphttp/websocket"
"github.com/yusing/goutils/http/httpheaders"
)
type (

View File

@@ -5,9 +5,9 @@ import (
"time"
"github.com/gin-gonic/gin"
"github.com/yusing/godoxy/internal/net/gphttp/httpheaders"
"github.com/yusing/godoxy/internal/net/gphttp/websocket"
"github.com/yusing/godoxy/internal/route/routes"
"github.com/yusing/goutils/http/httpheaders"
)
type HealthMap = map[string]routes.HealthInfo // @name HealthMap

View File

@@ -12,9 +12,9 @@ import (
"github.com/lithammer/fuzzysearch/fuzzy"
apitypes "github.com/yusing/godoxy/internal/api/types"
"github.com/yusing/godoxy/internal/homepage"
"github.com/yusing/godoxy/internal/net/gphttp/httpheaders"
"github.com/yusing/godoxy/internal/net/gphttp/websocket"
"github.com/yusing/godoxy/internal/route/routes"
"github.com/yusing/goutils/http/httpheaders"
)
type HomepageItemsRequest struct {

View File

@@ -17,9 +17,9 @@ import (
"github.com/yusing/godoxy/internal/gperr"
"github.com/yusing/godoxy/internal/metrics/period"
"github.com/yusing/godoxy/internal/metrics/systeminfo"
"github.com/yusing/godoxy/internal/net/gphttp/httpheaders"
"github.com/yusing/godoxy/internal/net/gphttp/websocket"
"github.com/yusing/godoxy/internal/utils/synk"
"github.com/yusing/goutils/http/httpheaders"
"github.com/yusing/goutils/synk"
)
var (

View File

@@ -10,7 +10,7 @@ import (
apitypes "github.com/yusing/godoxy/internal/api/types"
"github.com/yusing/godoxy/internal/metrics/period"
"github.com/yusing/godoxy/internal/metrics/systeminfo"
"github.com/yusing/godoxy/internal/net/gphttp/httpheaders"
"github.com/yusing/goutils/http/httpheaders"
)
type SystemInfoRequest struct {

View File

@@ -6,8 +6,8 @@ import (
"github.com/gin-gonic/gin"
config "github.com/yusing/godoxy/internal/config/types"
"github.com/yusing/godoxy/internal/net/gphttp/httpheaders"
"github.com/yusing/godoxy/internal/net/gphttp/websocket"
"github.com/yusing/goutils/http/httpheaders"
)
// @x-id "providers"

View File

@@ -6,11 +6,11 @@ import (
"time"
"github.com/gin-gonic/gin"
"github.com/yusing/godoxy/internal/net/gphttp/httpheaders"
"github.com/yusing/godoxy/internal/net/gphttp/websocket"
"github.com/yusing/godoxy/internal/route"
"github.com/yusing/godoxy/internal/route/routes"
"github.com/yusing/godoxy/internal/types"
"github.com/yusing/goutils/http/httpheaders"
)
type RouteType route.Route // @name Route

View File

@@ -6,9 +6,9 @@ import (
"github.com/gin-gonic/gin"
config "github.com/yusing/godoxy/internal/config/types"
"github.com/yusing/godoxy/internal/net/gphttp/httpheaders"
"github.com/yusing/godoxy/internal/net/gphttp/websocket"
"github.com/yusing/godoxy/internal/types"
"github.com/yusing/goutils/http/httpheaders"
)
type StatsResponse struct {

View File

@@ -10,7 +10,7 @@ import (
"github.com/yusing/godoxy/internal/common"
"github.com/yusing/godoxy/internal/gperr"
"github.com/yusing/godoxy/internal/net/gphttp"
"github.com/yusing/godoxy/internal/utils/strutils"
strutils "github.com/yusing/goutils/strings"
"golang.org/x/crypto/bcrypt"
)

View File

@@ -8,7 +8,7 @@ import (
"github.com/yusing/godoxy/internal/common"
"github.com/yusing/godoxy/internal/gperr"
"github.com/yusing/godoxy/internal/utils/strutils"
strutils "github.com/yusing/goutils/strings"
)
var (

View File

@@ -21,7 +21,7 @@ import (
"github.com/yusing/godoxy/internal/gperr"
"github.com/yusing/godoxy/internal/notif"
"github.com/yusing/godoxy/internal/task"
"github.com/yusing/godoxy/internal/utils/strutils"
strutils "github.com/yusing/goutils/strings"
)
type (

View File

@@ -5,7 +5,7 @@ import (
"os"
"github.com/rs/zerolog/log"
"github.com/yusing/godoxy/internal/utils/strutils"
strutils "github.com/yusing/goutils/strings"
)
func (p *Provider) Setup() (err error) {

View File

@@ -26,9 +26,9 @@ import (
proxy "github.com/yusing/godoxy/internal/route/provider"
"github.com/yusing/godoxy/internal/serialization"
"github.com/yusing/godoxy/internal/task"
"github.com/yusing/godoxy/internal/utils/strutils/ansi"
"github.com/yusing/godoxy/internal/watcher"
"github.com/yusing/godoxy/internal/watcher/events"
"github.com/yusing/goutils/strings/ansi"
)
type Config struct {

View File

@@ -147,8 +147,8 @@ require (
github.com/volcengine/volc-sdk-golang v1.0.221 // indirect
github.com/vultr/govultr/v3 v3.24.0 // indirect
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
github.com/yusing/godoxy/internal/utils v0.0.0 // indirect
github.com/yusing/goutils v0.0.0-20250922091446-1c6a11717d72 // indirect
github.com/yusing/godoxy/internal/utils v0.1.0 // indirect
github.com/yusing/goutils v0.2.1 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
go.opentelemetry.io/otel v1.38.0 // indirect

View File

@@ -1513,8 +1513,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yusing/goutils v0.0.0-20250922091446-1c6a11717d72 h1:NHYq8ZqoLSCJYzZaIOn8mJ28/Ac7GEFgeZgcfBoamWE=
github.com/yusing/goutils v0.0.0-20250922091446-1c6a11717d72/go.mod h1:XROGErdAT8UxsbLQbpejGhrTpknWEEgy+9HVjq7ULBI=
github.com/yusing/goutils v0.2.1 h1:KjoCrNO0otthaPCZPfQY+5GKsqs5+J77CxP+TNHYa/Y=
github.com/yusing/goutils v0.2.1/go.mod h1:v6RZsMRdzcts4udSg0vqUIFvaD0OaUMPTwYJZ4XnQYo=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=

View File

@@ -6,7 +6,7 @@ import (
"github.com/docker/docker/api/types/container"
"github.com/yusing/ds/ordered"
"github.com/yusing/godoxy/internal/types"
"github.com/yusing/godoxy/internal/utils/strutils"
strutils "github.com/yusing/goutils/strings"
)
type containerHelper struct {

View File

@@ -7,7 +7,7 @@ import (
"github.com/goccy/go-yaml"
"github.com/yusing/godoxy/internal/gperr"
"github.com/yusing/godoxy/internal/types"
"github.com/yusing/godoxy/internal/utils/strutils"
strutils "github.com/yusing/goutils/strings"
)
var ErrInvalidLabel = gperr.New("invalid label")

View File

@@ -5,8 +5,8 @@ import (
"strings"
"testing"
"github.com/yusing/godoxy/internal/utils/strutils/ansi"
expect "github.com/yusing/godoxy/internal/utils/testing"
"github.com/yusing/goutils/strings/ansi"
)
func TestBaseString(t *testing.T) {

View File

@@ -1,6 +1,6 @@
package gperr
import "github.com/yusing/godoxy/internal/utils/strutils/ansi"
import "github.com/yusing/goutils/strings/ansi"
type Hint struct {
Prefix string

View File

@@ -6,7 +6,7 @@ import (
"errors"
"slices"
"github.com/yusing/godoxy/internal/utils/strutils/ansi"
"github.com/yusing/goutils/strings/ansi"
)
//nolint:errname

View File

@@ -14,7 +14,7 @@ import (
"github.com/PuerkitoBio/goquery"
"github.com/vincent-petithory/dataurl"
gphttp "github.com/yusing/godoxy/internal/net/gphttp"
"github.com/yusing/godoxy/internal/utils/strutils"
strutils "github.com/yusing/goutils/strings"
)
type FetchResult struct {

View File

@@ -4,7 +4,7 @@ import (
"context"
"github.com/yusing/godoxy/internal/homepage/widgets"
"github.com/yusing/godoxy/internal/utils/strutils"
strutils "github.com/yusing/goutils/strings"
)
const endpointTransferInfo = "/api/v2/transfer/info"

View File

@@ -16,7 +16,7 @@ import (
"github.com/yusing/godoxy/internal/common"
"github.com/yusing/godoxy/internal/serialization"
"github.com/yusing/godoxy/internal/task"
"github.com/yusing/godoxy/internal/utils/strutils"
strutils "github.com/yusing/goutils/strings"
)
type (

View File

@@ -5,7 +5,7 @@ import (
"iter"
"strconv"
"github.com/yusing/godoxy/internal/utils/strutils"
strutils "github.com/yusing/goutils/strings"
)
type watcherDebug struct {

View File

@@ -7,7 +7,7 @@ import (
api "github.com/yusing/godoxy/internal/api/v1"
gphttp "github.com/yusing/godoxy/internal/net/gphttp"
"github.com/yusing/godoxy/internal/net/gphttp/httpheaders"
"github.com/yusing/goutils/http/httpheaders"
)
type ForceCacheControl struct {
@@ -107,7 +107,8 @@ func (w *Watcher) wakeFromHTTP(rw http.ResponseWriter, r *http.Request) (shouldN
w.l.Trace().Msg("signal received")
err := w.Wake(ctx)
if err != nil {
gphttp.ServerError(rw, r, err)
http.Error(rw, "Internal Server Error", http.StatusInternalServerError)
gphttp.LogError(r).Msg(fmt.Sprintf("failed to wake: %v", err))
return false
}

View File

@@ -5,7 +5,7 @@ import (
_ "embed"
"text/template"
"github.com/yusing/godoxy/internal/net/gphttp/httpheaders"
"github.com/yusing/goutils/http/httpheaders"
)
type templateData struct {

View File

@@ -15,7 +15,6 @@ import (
"github.com/yusing/godoxy/internal/gperr"
"github.com/yusing/godoxy/internal/idlewatcher/provider"
idlewatcher "github.com/yusing/godoxy/internal/idlewatcher/types"
"github.com/yusing/godoxy/internal/net/gphttp/reverseproxy"
nettypes "github.com/yusing/godoxy/internal/net/types"
"github.com/yusing/godoxy/internal/route/routes"
"github.com/yusing/godoxy/internal/task"
@@ -24,6 +23,7 @@ import (
"github.com/yusing/godoxy/internal/utils/atomic"
"github.com/yusing/godoxy/internal/watcher/events"
"github.com/yusing/godoxy/internal/watcher/health/monitor"
"github.com/yusing/goutils/http/reverseproxy"
"golang.org/x/sync/errgroup"
"golang.org/x/sync/singleflight"
)

View File

@@ -12,9 +12,9 @@ import (
"github.com/yusing/godoxy/internal/gperr"
maxmind "github.com/yusing/godoxy/internal/maxmind/types"
"github.com/yusing/godoxy/internal/task"
"github.com/yusing/godoxy/internal/utils"
"github.com/yusing/godoxy/internal/utils/strutils"
"github.com/yusing/godoxy/internal/utils/synk"
ioutils "github.com/yusing/goutils/io"
strutils "github.com/yusing/goutils/strings"
"github.com/yusing/goutils/synk"
"golang.org/x/time/rate"
)
@@ -26,7 +26,7 @@ type (
rawWriter io.Writer
closer []io.Closer
supportRotate []supportRotate
writer *utils.BufferedWriter
writer *ioutils.BufferedWriter
writeLock sync.Mutex
closed bool
@@ -119,7 +119,7 @@ func NewAccessLoggerWithIO(parent task.Parent, writer WriterWithName, anyCfg Any
task: parent.Subtask("accesslog."+writer.Name(), true),
cfg: cfg,
rawWriter: writer,
writer: utils.NewBufferedWriter(writer, MinBufferSize),
writer: ioutils.NewBufferedWriter(writer, MinBufferSize),
bufSize: MinBufferSize,
errRateLimiter: rate.NewLimiter(rate.Every(errRateLimit), errBurst),
logger: log.With().Str("file", writer.Name()).Logger(),

View File

@@ -10,8 +10,8 @@ import (
"github.com/spf13/afero"
"github.com/yusing/godoxy/internal/task"
"github.com/yusing/godoxy/internal/utils/strutils"
expect "github.com/yusing/godoxy/internal/utils/testing"
strutils "github.com/yusing/goutils/strings"
)
func TestBackScanner(t *testing.T) {

View File

@@ -7,7 +7,7 @@ import (
"github.com/yusing/godoxy/internal/gperr"
nettypes "github.com/yusing/godoxy/internal/net/types"
"github.com/yusing/godoxy/internal/utils/strutils"
strutils "github.com/yusing/goutils/strings"
)
type (

View File

@@ -7,8 +7,8 @@ import (
. "github.com/yusing/godoxy/internal/logging/accesslog"
nettypes "github.com/yusing/godoxy/internal/net/types"
"github.com/yusing/godoxy/internal/utils/strutils"
expect "github.com/yusing/godoxy/internal/utils/testing"
strutils "github.com/yusing/goutils/strings"
)
func TestStatusCodeFilter(t *testing.T) {

View File

@@ -5,7 +5,7 @@ import (
"strconv"
"github.com/yusing/godoxy/internal/gperr"
"github.com/yusing/godoxy/internal/utils/strutils"
strutils "github.com/yusing/goutils/strings"
)
type Retention struct {

View File

@@ -8,8 +8,8 @@ import (
"github.com/rs/zerolog"
"github.com/yusing/godoxy/internal/gperr"
"github.com/yusing/godoxy/internal/utils"
"github.com/yusing/godoxy/internal/utils/strutils"
"github.com/yusing/godoxy/internal/utils/synk"
strutils "github.com/yusing/goutils/strings"
"github.com/yusing/goutils/synk"
)
type supportRotate interface {

View File

@@ -9,8 +9,8 @@ import (
. "github.com/yusing/godoxy/internal/logging/accesslog"
"github.com/yusing/godoxy/internal/task"
"github.com/yusing/godoxy/internal/utils"
"github.com/yusing/godoxy/internal/utils/strutils"
expect "github.com/yusing/godoxy/internal/utils/testing"
strutils "github.com/yusing/goutils/strings"
)
var (

View File

@@ -4,7 +4,7 @@ import (
"strconv"
"github.com/yusing/godoxy/internal/gperr"
"github.com/yusing/godoxy/internal/utils/strutils"
strutils "github.com/yusing/goutils/strings"
)
type StatusCodeRange struct {

View File

@@ -8,7 +8,7 @@ import (
"github.com/rs/zerolog"
"github.com/yusing/godoxy/internal/common"
"github.com/yusing/godoxy/internal/utils/strutils"
strutils "github.com/yusing/goutils/strings"
zerologlog "github.com/rs/zerolog/log"
)

View File

@@ -8,8 +8,8 @@ import (
"github.com/gin-gonic/gin"
apitypes "github.com/yusing/godoxy/internal/api/types"
metricsutils "github.com/yusing/godoxy/internal/metrics/utils"
"github.com/yusing/godoxy/internal/net/gphttp/httpheaders"
"github.com/yusing/godoxy/internal/net/gphttp/websocket"
"github.com/yusing/goutils/http/httpheaders"
)
type ResponseType[AggregateT any] struct {

View File

@@ -1,69 +0,0 @@
package httpheaders
import (
"net/http"
"strings"
)
// AppendCSP appends a CSP header to specific directives in the response writer.
//
// Directives other than the ones in cspDirectives will be kept as is.
//
// It will replace 'none' with the sources.
//
// It will append 'self' to the sources if it's not already present.
func AppendCSP(w http.ResponseWriter, r *http.Request, cspDirectives []string, sources []string) {
csp := make(map[string]string)
cspValues := r.Header.Values("Content-Security-Policy")
if len(cspValues) == 1 {
cspValues = strings.Split(cspValues[0], ";")
for i, cspString := range cspValues {
cspValues[i] = strings.TrimSpace(cspString)
}
}
for _, cspString := range cspValues {
parts := strings.SplitN(cspString, " ", 2)
if len(parts) == 2 {
csp[parts[0]] = parts[1]
}
}
for _, directive := range cspDirectives {
value, ok := csp[directive]
if !ok {
value = "'self'"
}
switch value {
case "'self'":
csp[directive] = value + " " + strings.Join(sources, " ")
case "'none'":
csp[directive] = strings.Join(sources, " ")
default:
for _, source := range sources {
if !strings.Contains(value, source) {
value += " " + source
}
}
if !strings.Contains(value, "'self'") {
value = "'self' " + value
}
csp[directive] = value
}
}
values := make([]string, 0, len(csp))
for directive, value := range csp {
values = append(values, directive+" "+value)
}
// Remove existing CSP header, case insensitive
for k := range w.Header() {
if strings.EqualFold(k, "Content-Security-Policy") {
delete(w.Header(), k)
}
}
// Set new CSP header
w.Header()["Content-Security-Policy"] = values
}

View File

@@ -1,168 +0,0 @@
package httpheaders
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
)
func TestAppendCSP(t *testing.T) {
tests := []struct {
name string
initialHeaders map[string][]string
sources []string
directives []string
expectedCSP map[string]string
}{
{
name: "No CSP header",
initialHeaders: map[string][]string{},
sources: []string{},
directives: []string{"default-src", "script-src", "frame-src", "style-src", "connect-src"},
expectedCSP: map[string]string{"default-src": "'self'", "script-src": "'self'", "frame-src": "'self'", "style-src": "'self'", "connect-src": "'self'"},
},
{
name: "No CSP header with sources",
initialHeaders: map[string][]string{},
sources: []string{"https://example.com"},
directives: []string{"default-src", "script-src", "frame-src", "style-src", "connect-src"},
expectedCSP: map[string]string{"default-src": "'self' https://example.com", "script-src": "'self' https://example.com", "frame-src": "'self' https://example.com", "style-src": "'self' https://example.com", "connect-src": "'self' https://example.com"},
},
{
name: "replace 'none' with sources",
initialHeaders: map[string][]string{
"Content-Security-Policy": {"default-src 'none'"},
},
sources: []string{"https://example.com"},
directives: []string{"default-src"},
expectedCSP: map[string]string{"default-src": "https://example.com"},
},
{
name: "CSP header with some directives",
initialHeaders: map[string][]string{
"Content-Security-Policy": {"default-src 'none'", "script-src 'unsafe-inline'"},
},
sources: []string{"https://example.com"},
directives: []string{"script-src"},
expectedCSP: map[string]string{
"default-src": "'none",
"script-src": "'unsafe-inline' https://example.com",
},
},
{
name: "CSP header with some directives with self",
initialHeaders: map[string][]string{
"Content-Security-Policy": {"default-src 'self'", "connect-src 'self'"},
},
sources: []string{"https://api.example.com"},
directives: []string{"default-src", "connect-src"},
expectedCSP: map[string]string{
"default-src": "'self' https://api.example.com",
"connect-src": "'self' https://api.example.com",
},
},
{
name: "AppendCSP sources conflict with existing CSP header",
initialHeaders: map[string][]string{
"Content-Security-Policy": {"default-src 'self' https://cdn.example.com", "script-src 'unsafe-inline'"},
},
sources: []string{"https://cdn.example.com", "https://api.example.com"},
directives: []string{"default-src", "script-src"},
expectedCSP: map[string]string{
"default-src": "'self' https://cdn.example.com https://api.example.com",
"script-src": "'unsafe-inline' https://cdn.example.com https://api.example.com",
},
},
{
name: "Non-standard CSP directive",
initialHeaders: map[string][]string{
"Content-Security-Policy": {
"default-src 'self'",
"script-src 'unsafe-inline'",
"img-src 'self'", // img-src is not in cspDirectives list
},
},
sources: []string{"https://example.com"},
directives: []string{"default-src", "script-src"},
expectedCSP: map[string]string{
"default-src": "'self' https://example.com",
"script-src": "'unsafe-inline' https://example.com",
// img-src should not be present in response as it's not in cspDirectives
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
// Create a test request with initial headers
req := httptest.NewRequest(http.MethodGet, "/", nil)
for header, values := range tc.initialHeaders {
req.Header[header] = values
}
// Create a test response recorder
w := httptest.NewRecorder()
// Call the function under test
AppendCSP(w, req, tc.directives, tc.sources)
// Check the resulting CSP headers
respHeaders := w.Header()
cspValues, exists := respHeaders["Content-Security-Policy"]
// If we expect no CSP headers, verify none exist
if len(tc.expectedCSP) == 0 {
if exists && len(cspValues) > 0 {
t.Errorf("Expected no CSP header, but got %v", cspValues)
}
return
}
// Verify CSP headers exist when expected
if !exists || len(cspValues) == 0 {
t.Errorf("Expected CSP header to be set, but it was not")
return
}
// Parse the CSP response and verify each directive
foundDirectives := make(map[string]string)
for _, cspValue := range cspValues {
parts := strings.Split(cspValue, ";")
for _, part := range parts {
part = strings.TrimSpace(part)
if part == "" {
continue
}
directiveParts := strings.SplitN(part, " ", 2)
if len(directiveParts) != 2 {
t.Errorf("Invalid CSP directive format: %s", part)
continue
}
directive := directiveParts[0]
value := directiveParts[1]
foundDirectives[directive] = value
}
}
// Verify expected directives
for directive, expectedValue := range tc.expectedCSP {
actualValue, ok := foundDirectives[directive]
if !ok {
t.Errorf("Expected directive %s not found in response", directive)
continue
}
// Check if all expected sources are in the actual value
expectedSources := strings.SplitSeq(expectedValue, " ")
for source := range expectedSources {
if !strings.Contains(actualValue, source) {
t.Errorf("Directive %s missing expected source %s. Got: %s", directive, source, actualValue)
}
}
}
})
}
}

View File

@@ -1,119 +0,0 @@
package httpheaders
import (
"net/http"
"net/textproto"
"strings"
"golang.org/x/net/http/httpguts"
)
const (
HeaderXForwardedMethod = "X-Forwarded-Method"
HeaderXForwardedFor = "X-Forwarded-For"
HeaderXForwardedProto = "X-Forwarded-Proto"
HeaderXForwardedHost = "X-Forwarded-Host"
HeaderXForwardedPort = "X-Forwarded-Port"
HeaderXForwardedURI = "X-Forwarded-Uri"
HeaderXRealIP = "X-Real-IP"
HeaderContentType = "Content-Type"
HeaderContentLength = "Content-Length"
HeaderGoDoxyCheckRedirect = "X-Godoxy-Check-Redirect"
)
// Hop-by-hop headers. These are removed when sent to the backend.
// As of RFC 7230, hop-by-hop headers are required to appear in the
// Connection header field. These are the headers defined by the
// obsoleted RFC 2616 (section 13.5.1) and are used for backward
// compatibility.
var hopHeaders = []string{
"Connection",
"Proxy-Connection", // non-standard but still sent by libcurl and rejected by e.g. google
"Keep-Alive",
"Proxy-Authenticate",
"Proxy-Authorization",
"Te", // canonicalized version of "TE"
"Trailer", // not Trailers per URL above; https://www.rfc-editor.org/errata_search.php?eid=4522
"Transfer-Encoding",
"Upgrade",
}
func UpgradeType(h http.Header) string {
if !httpguts.HeaderValuesContainsToken(h["Connection"], "Upgrade") {
return ""
}
return h.Get("Upgrade")
}
// RemoveHopByHopHeaders removes hop-by-hop headers.
func RemoveHopByHopHeaders(h http.Header) {
// RFC 7230, section 6.1: Remove headers listed in the "Connection" header.
for _, f := range h["Connection"] {
for sf := range strings.SplitSeq(f, ",") {
if sf = textproto.TrimString(sf); sf != "" {
h.Del(sf)
}
}
}
// RFC 2616, section 13.5.1: Remove a set of known hop-by-hop headers.
// This behavior is superseded by the RFC 7230 Connection header, but
// preserve it for backwards compatibility.
for _, f := range hopHeaders {
h.Del(f)
}
}
func RemoveHop(h http.Header) {
reqUpType := UpgradeType(h)
RemoveHopByHopHeaders(h)
if reqUpType != "" {
h.Set("Connection", "Upgrade")
h.Set("Upgrade", reqUpType)
} else {
h.Del("Connection")
}
}
func RemoveServiceHeaders(h http.Header) {
h.Del("X-Powered-By")
h.Del("Server")
}
func CopyHeader(dst, src http.Header) {
for k, vv := range src {
for _, v := range vv {
dst.Add(k, v)
}
}
}
func FilterHeaders(h http.Header, allowed []string) http.Header {
if len(allowed) == 0 {
return h
}
filtered := make(http.Header)
for i, header := range allowed {
values := h.Values(header)
if len(values) == 0 {
continue
}
filtered[http.CanonicalHeaderKey(allowed[i])] = append([]string(nil), values...)
}
return filtered
}
func HeaderToMap(h http.Header) map[string]string {
result := make(map[string]string)
for k, v := range h {
if len(v) > 0 {
result[k] = v[0] // Take the first value
}
}
return result
}

View File

@@ -1,9 +0,0 @@
package httpheaders
import (
"net/http"
)
func IsWebsocket(h http.Header) bool {
return UpgradeType(h) == "websocket"
}

View File

@@ -9,10 +9,10 @@ import (
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/yusing/godoxy/internal/gperr"
"github.com/yusing/godoxy/internal/net/gphttp/httpheaders"
"github.com/yusing/godoxy/internal/task"
"github.com/yusing/godoxy/internal/types"
"github.com/yusing/godoxy/internal/utils/pool"
"github.com/yusing/goutils/http/httpheaders"
)
// TODO: stats of each server.

View File

@@ -12,12 +12,11 @@ import (
"github.com/yusing/godoxy/internal/entrypoint"
. "github.com/yusing/godoxy/internal/net/gphttp/middleware"
"github.com/yusing/godoxy/internal/net/gphttp/reverseproxy"
nettypes "github.com/yusing/godoxy/internal/net/types"
"github.com/yusing/godoxy/internal/route"
routeTypes "github.com/yusing/godoxy/internal/route/types"
"github.com/yusing/godoxy/internal/task"
expect "github.com/yusing/godoxy/internal/utils/testing"
"github.com/yusing/goutils/http/reverseproxy"
)
func noOpHandler(w http.ResponseWriter, r *http.Request) {}
@@ -102,8 +101,10 @@ func (f fakeRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
}
func TestReverseProxyBypass(t *testing.T) {
rp := reverseproxy.NewReverseProxy("test", nettypes.MustParseURL("http://example.com"), fakeRoundTripper{})
err := PatchReverseProxy(rp, map[string]OptionsRaw{
url, err := url.Parse("http://example.com")
expect.NoError(t, err)
rp := reverseproxy.NewReverseProxy("test", url, fakeRoundTripper{})
err = PatchReverseProxy(rp, map[string]OptionsRaw{
"response": {
"bypass": "path /test/* | path /api",
"set_headers": map[string]string{

View File

@@ -15,7 +15,7 @@ import (
"github.com/yusing/godoxy/internal/common"
nettypes "github.com/yusing/godoxy/internal/net/types"
"github.com/yusing/godoxy/internal/utils/atomic"
"github.com/yusing/godoxy/internal/utils/strutils"
strutils "github.com/yusing/goutils/strings"
)
type cloudflareRealIP struct {

View File

@@ -10,8 +10,8 @@ import (
"github.com/rs/zerolog/log"
gphttp "github.com/yusing/godoxy/internal/net/gphttp"
"github.com/yusing/godoxy/internal/net/gphttp/httpheaders"
"github.com/yusing/godoxy/internal/net/gphttp/middleware/errorpage"
"github.com/yusing/goutils/http/httpheaders"
)
type customErrorPage struct{}

View File

@@ -7,9 +7,9 @@ import (
"net/http"
"time"
"github.com/yusing/godoxy/internal/net/gphttp/httpheaders"
"github.com/yusing/godoxy/internal/route/routes"
"github.com/yusing/godoxy/internal/utils"
httputils "github.com/yusing/goutils/http"
"github.com/yusing/goutils/http/httpheaders"
)
type (
@@ -91,7 +91,7 @@ func (m *forwardAuthMiddleware) before(w http.ResponseWriter, r *http.Request) (
defer resp.Body.Close()
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
body, release, err := utils.ReadAllBody(resp)
body, release, err := httputils.ReadAllBody(resp)
defer release()
if err != nil {

View File

@@ -12,8 +12,8 @@ import (
"github.com/rs/zerolog/log"
"github.com/yusing/godoxy/internal/gperr"
gphttp "github.com/yusing/godoxy/internal/net/gphttp"
"github.com/yusing/godoxy/internal/net/gphttp/reverseproxy"
"github.com/yusing/godoxy/internal/serialization"
"github.com/yusing/goutils/http/reverseproxy"
)
type (

View File

@@ -9,7 +9,7 @@ import (
"github.com/yusing/godoxy/internal/common"
"github.com/yusing/godoxy/internal/gperr"
"github.com/yusing/godoxy/internal/utils"
"github.com/yusing/godoxy/internal/utils/strutils"
strutils "github.com/yusing/goutils/strings"
)
// snakes and cases will be stripped on `Get`

View File

@@ -9,8 +9,9 @@ import (
"github.com/PuerkitoBio/goquery"
"github.com/rs/zerolog/log"
gphttp "github.com/yusing/godoxy/internal/net/gphttp"
"github.com/yusing/godoxy/internal/utils"
"github.com/yusing/godoxy/internal/utils/synk"
httputils "github.com/yusing/goutils/http"
ioutils "github.com/yusing/goutils/io"
"github.com/yusing/goutils/synk"
"golang.org/x/net/html"
)
@@ -40,7 +41,7 @@ func (m *modifyHTML) modifyResponse(resp *http.Response) error {
}
// NOTE: do not put it in the defer, it will be used as resp.Body
content, release, err := utils.ReadAllBody(resp)
content, release, err := httputils.ReadAllBody(resp)
if err != nil {
resp.Body.Close()
return err
@@ -71,19 +72,19 @@ func (m *modifyHTML) modifyResponse(resp *http.Response) error {
}
buf := bytes.NewBuffer(content[:0])
err = buildHTML(m, doc, buf)
err = buildHTML(doc, buf)
if err != nil {
return err
}
resp.ContentLength = int64(buf.Len())
resp.Header.Set("Content-Length", strconv.Itoa(buf.Len()))
resp.Header.Set("Content-Type", "text/html; charset=utf-8")
resp.Body = utils.NewHookCloser(io.NopCloser(bytes.NewReader(buf.Bytes())), release)
resp.Body = ioutils.NewHookReadCloser(io.NopCloser(bytes.NewReader(buf.Bytes())), release)
return nil
}
// copied and modified from (*goquery.Selection).Html()
func buildHTML(m *modifyHTML, s *goquery.Document, buf *bytes.Buffer) error {
func buildHTML(s *goquery.Document, buf *bytes.Buffer) error {
// Merge all head nodes into one
headNodes := s.Find("head")
if headNodes.Length() > 1 {

View File

@@ -4,8 +4,8 @@ import (
"net"
"net/http"
"github.com/yusing/godoxy/internal/net/gphttp/httpheaders"
nettypes "github.com/yusing/godoxy/internal/net/types"
"github.com/yusing/goutils/http/httpheaders"
)
// https://nginx.org/en/docs/http/ngx_http_realip_module.html

View File

@@ -6,9 +6,9 @@ import (
"strings"
"testing"
"github.com/yusing/godoxy/internal/net/gphttp/httpheaders"
nettypes "github.com/yusing/godoxy/internal/net/types"
. "github.com/yusing/godoxy/internal/utils/testing"
"github.com/yusing/goutils/http/httpheaders"
)
func TestSetRealIPOpts(t *testing.T) {

View File

@@ -11,9 +11,9 @@ import (
"github.com/yusing/godoxy/internal/common"
"github.com/yusing/godoxy/internal/gperr"
"github.com/yusing/godoxy/internal/net/gphttp/reverseproxy"
nettypes "github.com/yusing/godoxy/internal/net/types"
. "github.com/yusing/godoxy/internal/utils/testing"
"github.com/yusing/goutils/http/reverseproxy"
)
//go:embed test_data/sample_headers.json
@@ -152,7 +152,7 @@ func newMiddlewaresTest(middlewares []*Middleware, args *testArgs) (*TestResult,
rr.parent = http.DefaultTransport
}
rp := reverseproxy.NewReverseProxy("test", args.upstreamURL, rr)
rp := reverseproxy.NewReverseProxy("test", &args.upstreamURL.URL, rr)
patchReverseProxy(rp, middlewares)
rp.ServeHTTP(w, req)

View File

@@ -5,7 +5,7 @@ import (
"net/http"
"strings"
"github.com/yusing/godoxy/internal/net/gphttp/httpheaders"
"github.com/yusing/goutils/http/httpheaders"
)
type (

View File

@@ -1,560 +0,0 @@
// Copyright 2011 The Go Authors.
// Modified from the Go project under the a BSD-style License (https://cs.opensource.google/go/go/+/refs/tags/go1.23.1:src/net/http/httputil/reverseproxy.go)
// https://cs.opensource.google/go/go/+/master:LICENSE
package reverseproxy
// This is a small mod on net/http/httputil/reverseproxy.go
// that boosts performance in some cases
// and compatible to other modules of this project
// Copyright (c) 2024 yusing
import (
"bytes"
"context"
"crypto/tls"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/http/httptrace"
"net/textproto"
"net/url"
"strings"
"sync"
"github.com/quic-go/quic-go/http3"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/yusing/godoxy/internal/logging/accesslog"
"github.com/yusing/godoxy/internal/net/gphttp/httpheaders"
nettypes "github.com/yusing/godoxy/internal/net/types"
U "github.com/yusing/godoxy/internal/utils"
"golang.org/x/net/http/httpguts"
"golang.org/x/net/http2"
_ "unsafe"
)
// A ProxyRequest contains a request to be rewritten by a [ReverseProxy].
type ProxyRequest struct {
// In is the request received by the proxy.
// The Rewrite function must not modify In.
In *http.Request
// Out is the request which will be sent by the proxy.
// The Rewrite function may modify or replace this request.
// Hop-by-hop headers are removed from this request
// before Rewrite is called.
Out *http.Request
}
// SetXForwarded sets the X-Forwarded-For, X-Forwarded-Host, and
// X-Forwarded-Proto headers of the outbound request.
//
// - The X-Forwarded-For header is set to the client IP address.
// - The X-Forwarded-Host header is set to the host name requested
// by the client.
// - The X-Forwarded-Proto header is set to "http" or "https", depending
// on whether the inbound request was made on a TLS-enabled connection.
//
// If the outbound request contains an existing X-Forwarded-For header,
// SetXForwarded appends the client IP address to it. To append to the
// inbound request's X-Forwarded-For header (the default behavior of
// [ReverseProxy] when using a Director function), copy the header
// from the inbound request before calling SetXForwarded:
//
// rewriteFunc := func(r *httputil.ProxyRequest) {
// r.Out.Header["X-Forwarded-For"] = r.In.Header["X-Forwarded-For"]
// r.SetXForwarded()
// }
// ReverseProxy is an HTTP Handler that takes an incoming request and
// sends it to another server, proxying the response back to the
// client.
//
// 1xx responses are forwarded to the client if the underlying
// transport supports ClientTrace.Got1xxResponse.
type ReverseProxy struct {
zerolog.Logger
// The transport used to perform proxy requests.
Transport http.RoundTripper
// ModifyResponse is an optional function that modifies the
// Response from the backend. It is called if the backend
// returns a response at all, with any HTTP status code.
// If the backend is unreachable, the optional ErrorHandler is
// called before ModifyResponse.
//
// If ModifyResponse returns an error, ErrorHandler is called
// with its error value. If ErrorHandler is nil, its default
// implementation is used.
ModifyResponse func(*http.Response) error
AccessLogger *accesslog.AccessLogger
HandlerFunc http.HandlerFunc
TargetName string
TargetURL *nettypes.URL
}
func singleJoiningSlash(a, b string) string {
aslash := strings.HasSuffix(a, "/")
bslash := strings.HasPrefix(b, "/")
switch {
case aslash && bslash:
return a + b[1:]
case !aslash && !bslash:
return a + "/" + b
}
return a + b
}
func joinURLPath(a, b *url.URL) (path, rawpath string) {
if a.RawPath == "" && b.RawPath == "" {
return singleJoiningSlash(a.Path, b.Path), ""
}
// Same as singleJoiningSlash, but uses EscapedPath to determine
// whether a slash should be added
apath := a.EscapedPath()
bpath := b.EscapedPath()
aslash := strings.HasSuffix(apath, "/")
bslash := strings.HasPrefix(bpath, "/")
switch {
case aslash && bslash:
return a.Path + b.Path[1:], apath + bpath[1:]
case !aslash && !bslash:
return a.Path + "/" + b.Path, apath + "/" + bpath
}
return a.Path + b.Path, apath + bpath
}
// NewReverseProxy returns a new [ReverseProxy] that routes
// URLs to the scheme, host, and base path provided in target. If the
// target's path is "/base" and the incoming request was for "/dir",
// the target request will be for /base/dir.
func NewReverseProxy(name string, target *nettypes.URL, transport http.RoundTripper) *ReverseProxy {
if transport == nil {
panic("nil transport")
}
rp := &ReverseProxy{
Logger: log.With().Str("name", name).Logger(),
Transport: transport,
TargetName: name,
TargetURL: target,
}
rp.HandlerFunc = rp.handler
return rp
}
func (p *ReverseProxy) rewriteRequestURL(req *http.Request) {
targetQuery := p.TargetURL.RawQuery
req.URL.Scheme = p.TargetURL.Scheme
req.URL.Host = p.TargetURL.Host
req.URL.Path, req.URL.RawPath = joinURLPath(&p.TargetURL.URL, req.URL)
if targetQuery == "" || req.URL.RawQuery == "" {
req.URL.RawQuery = targetQuery + req.URL.RawQuery
} else {
req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery
}
}
func copyHeader(dst, src http.Header) {
for k, vv := range src {
for _, v := range vv {
dst.Add(k, v)
}
}
}
//go:linkname errStreamClosed golang.org/x/net/http2.errStreamClosed
var errStreamClosed error
func (p *ReverseProxy) errorHandler(rw http.ResponseWriter, r *http.Request, err error, writeHeader bool) {
reqURL := r.Host + r.URL.Path
switch {
case errors.Is(err, context.Canceled), errors.Is(err, io.EOF):
log.Trace().Err(err).Str("url", reqURL).Msg("http proxy error")
case errors.Is(err, context.DeadlineExceeded):
log.Debug().Err(err).Str("url", reqURL).Msg("http proxy error")
default:
var recordErr tls.RecordHeaderError
if errors.As(err, &recordErr) {
log.Error().
Str("url", reqURL).
Msgf(`scheme was likely misconfigured as https,
try setting "proxy.%s.scheme" back to "http"`, p.TargetName)
log.Err(err).Msg("underlying error")
goto logged
}
if errors.Is(err, errStreamClosed) {
goto logged
}
var h2Err http2.StreamError
if errors.As(err, &h2Err) {
// ignore these errors
switch h2Err.Code {
case http2.ErrCodeStreamClosed:
goto logged
}
}
var h3Err *http3.Error
if errors.As(err, &h3Err) {
// ignore these errors
switch h3Err.ErrorCode {
case
http3.ErrCodeNoError,
http3.ErrCodeRequestCanceled:
goto logged
}
}
log.Err(err).Str("url", reqURL).Msg("http proxy error")
}
logged:
if writeHeader {
rw.WriteHeader(http.StatusInternalServerError)
}
if p.AccessLogger != nil {
p.AccessLogger.LogError(r, err)
}
}
// modifyResponse conditionally runs the optional ModifyResponse hook
// and reports whether the request should proceed.
func (p *ReverseProxy) modifyResponse(rw http.ResponseWriter, res *http.Response, origReq, req *http.Request) bool {
if p.ModifyResponse == nil {
return true
}
res.Request = origReq
err := p.ModifyResponse(res)
res.Request = req
if err != nil {
res.Body.Close()
p.errorHandler(rw, req, err, true)
return false
}
return true
}
func (p *ReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
p.HandlerFunc(rw, req)
}
func (p *ReverseProxy) handler(rw http.ResponseWriter, req *http.Request) {
transport := p.Transport
ctx := req.Context()
if ctx.Done() != nil {
// CloseNotifier predates context.Context, and has been
// entirely superseded by it. If the request contains
// a Context that carries a cancellation signal, don't
// bother spinning up a goroutine to watch the CloseNotify
// channel (if any).
//
// If the request Context has a nil Done channel (which
// means it is either context.Background, or a custom
// Context implementation with no cancellation signal),
// then consult the CloseNotifier if available.
} else if cn, ok := rw.(http.CloseNotifier); ok {
var cancel context.CancelFunc
ctx, cancel = context.WithCancel(ctx)
defer cancel()
notifyChan := cn.CloseNotify()
go func() {
select {
case <-notifyChan:
cancel()
case <-ctx.Done():
}
}()
}
outreq := req.Clone(ctx)
if req.ContentLength == 0 {
outreq.Body = nil // Issue 16036: nil Body for http.Transport retries
}
if outreq.Body != nil {
// Reading from the request body after returning from a handler is not
// allowed, and the RoundTrip goroutine that reads the Body can outlive
// this handler. This can lead to a crash if the handler panics (see
// Issue 46866). Although calling Close doesn't guarantee there isn't
// any Read in flight after the handle returns, in practice it's safe to
// read after closing it.
defer outreq.Body.Close()
}
if outreq.Header == nil {
outreq.Header = make(http.Header) // Issue 33142: historical behavior was to always allocate
}
p.rewriteRequestURL(outreq)
outreq.Close = false
reqUpType := httpheaders.UpgradeType(outreq.Header)
if !IsPrint(reqUpType) {
p.errorHandler(rw, req, fmt.Errorf("client tried to switch to invalid protocol %q", reqUpType), true)
return
}
outreq.Header.Del("Forwarded")
httpheaders.RemoveHopByHopHeaders(outreq.Header)
// Issue 21096: tell backend applications that care about trailer support
// that we support trailers. (We do, but we don't go out of our way to
// advertise that unless the incoming client request thought it was worth
// mentioning.) Note that we look at req.Header, not outreq.Header, since
// the latter has passed through removeHopByHopHeaders.
if httpguts.HeaderValuesContainsToken(req.Header["Te"], "trailers") {
outreq.Header.Set("Te", "trailers")
}
// After stripping all the hop-by-hop connection headers above, add back any
// necessary for protocol upgrades, such as for websockets.
if reqUpType != "" {
outreq.Header.Set("Connection", "Upgrade")
outreq.Header.Set("Upgrade", reqUpType)
if strings.EqualFold(reqUpType, "websocket") {
cleanWebsocketHeaders(outreq)
}
}
// If we aren't the first proxy retain prior
// X-Forwarded-For information as a comma+space
// separated list and fold multiple headers into one.
prior, ok := outreq.Header[httpheaders.HeaderXForwardedFor]
omit := ok && prior == nil // Issue 38079: nil now means don't populate the header
if !omit {
xff, _, err := net.SplitHostPort(req.RemoteAddr)
if err != nil {
xff = req.RemoteAddr
}
if len(prior) > 0 {
xff = strings.Join(prior, ", ") + ", " + xff
}
outreq.Header.Set(httpheaders.HeaderXForwardedFor, xff)
}
var reqScheme string
if req.TLS != nil || req.Header.Get("X-Forwarded-Proto") == "https" {
reqScheme = "https"
} else {
reqScheme = "http"
}
outreq.Header.Set(httpheaders.HeaderXForwardedMethod, req.Method)
outreq.Header.Set(httpheaders.HeaderXForwardedProto, reqScheme)
outreq.Header.Set(httpheaders.HeaderXForwardedHost, req.Host)
outreq.Header.Set(httpheaders.HeaderXForwardedURI, req.RequestURI)
if _, ok := outreq.Header["User-Agent"]; !ok {
// If the outbound request doesn't have a User-Agent header set,
// don't send the default Go HTTP client User-Agent.
outreq.Header.Set("User-Agent", "")
}
var (
roundTripMutex sync.Mutex
roundTripDone bool
)
trace := &httptrace.ClientTrace{
Got1xxResponse: func(code int, header textproto.MIMEHeader) error {
roundTripMutex.Lock()
defer roundTripMutex.Unlock()
if roundTripDone {
// If RoundTrip has returned, don't try to further modify
// the ResponseWriter's header map.
return nil
}
h := rw.Header()
copyHeader(h, http.Header(header))
rw.WriteHeader(code)
// Clear headers, it's not automatically done by ResponseWriter.WriteHeader() for 1xx responses
clear(h)
return nil
},
}
outreq = outreq.WithContext(httptrace.WithClientTrace(outreq.Context(), trace)) //nolint:contextcheck
res, err := transport.RoundTrip(outreq)
roundTripMutex.Lock()
roundTripDone = true
roundTripMutex.Unlock()
if err != nil {
p.errorHandler(rw, outreq, err, false)
res = &http.Response{
Status: http.StatusText(http.StatusBadGateway),
StatusCode: http.StatusBadGateway,
Proto: req.Proto,
ProtoMajor: req.ProtoMajor,
ProtoMinor: req.ProtoMinor,
Header: http.Header{},
Body: io.NopCloser(bytes.NewReader([]byte("Origin server is not reachable."))),
Request: req,
TLS: req.TLS,
}
}
if p.AccessLogger != nil {
defer func() {
p.AccessLogger.Log(req, res)
}()
}
httpheaders.RemoveServiceHeaders(res.Header)
// Deal with 101 Switching Protocols responses: (WebSocket, h2c, etc)
if res.StatusCode == http.StatusSwitchingProtocols {
if !p.modifyResponse(rw, res, req, outreq) {
return
}
p.handleUpgradeResponse(rw, outreq, res)
return
}
httpheaders.RemoveHopByHopHeaders(res.Header)
if !p.modifyResponse(rw, res, req, outreq) {
return
}
copyHeader(rw.Header(), res.Header)
// The "Trailer" header isn't included in the Transport's response,
// at least for *http.Transport. Build it up from Trailer.
announcedTrailers := len(res.Trailer)
if announcedTrailers > 0 {
trailerKeys := make([]string, 0, len(res.Trailer))
for k := range res.Trailer {
trailerKeys = append(trailerKeys, k)
}
rw.Header().Add("Trailer", strings.Join(trailerKeys, ", "))
}
rw.WriteHeader(res.StatusCode)
err = U.CopyCloseWithContext(ctx, rw, res.Body, int(res.ContentLength)) // close now, instead of defer, to populate res.Trailer
if err != nil {
if !errors.Is(err, context.Canceled) {
p.errorHandler(rw, req, err, false)
}
return
}
if len(res.Trailer) > 0 {
// Force chunking if we saw a response trailer.
// This prevents net/http from calculating the length for short
// bodies and adding a Content-Length.
http.NewResponseController(rw).Flush()
}
if len(res.Trailer) == announcedTrailers {
copyHeader(rw.Header(), res.Trailer)
return
}
for k, vv := range res.Trailer {
k = http.TrailerPrefix + k
for _, v := range vv {
rw.Header().Add(k, v)
}
}
}
// reference: https://github.com/traefik/traefik/blob/master/pkg/proxy/httputil/proxy.go
// https://tools.ietf.org/html/rfc6455#page-20
func cleanWebsocketHeaders(req *http.Request) {
req.Header["Sec-WebSocket-Key"] = req.Header["Sec-Websocket-Key"]
delete(req.Header, "Sec-Websocket-Key")
req.Header["Sec-WebSocket-Extensions"] = req.Header["Sec-Websocket-Extensions"]
delete(req.Header, "Sec-Websocket-Extensions")
req.Header["Sec-WebSocket-Accept"] = req.Header["Sec-Websocket-Accept"]
delete(req.Header, "Sec-Websocket-Accept")
req.Header["Sec-WebSocket-Protocol"] = req.Header["Sec-Websocket-Protocol"]
delete(req.Header, "Sec-Websocket-Protocol")
req.Header["Sec-WebSocket-Version"] = req.Header["Sec-Websocket-Version"]
delete(req.Header, "Sec-Websocket-Version")
}
func (p *ReverseProxy) handleUpgradeResponse(rw http.ResponseWriter, req *http.Request, res *http.Response) {
reqUpType := httpheaders.UpgradeType(req.Header)
resUpType := httpheaders.UpgradeType(res.Header)
if !IsPrint(resUpType) { // We know reqUpType is ASCII, it's checked by the caller.
p.errorHandler(rw, req, fmt.Errorf("backend tried to switch to invalid protocol %q", resUpType), true)
return
}
if !strings.EqualFold(reqUpType, resUpType) {
p.errorHandler(rw, req, fmt.Errorf("backend tried to switch protocol %q when %q was requested", resUpType, reqUpType), true)
return
}
backConn, ok := res.Body.(io.ReadWriteCloser)
if !ok {
p.errorHandler(rw, req, errors.New("internal error: 101 switching protocols response with non-writable body"), true)
return
}
rc := http.NewResponseController(rw)
conn, brw, hijackErr := rc.Hijack()
if errors.Is(hijackErr, http.ErrNotSupported) {
p.errorHandler(rw, req, fmt.Errorf("can't switch protocols using non-Hijacker ResponseWriter type %T", rw), true)
return
}
backConnCloseCh := make(chan bool)
go func() {
// Ensure that the cancellation of a request closes the backend.
// See issue https://golang.org/issue/35559.
select {
case <-req.Context().Done():
case <-backConnCloseCh:
}
backConn.Close()
}()
defer close(backConnCloseCh)
if hijackErr != nil {
p.errorHandler(rw, req, fmt.Errorf("hijack failed on protocol switch: %w", hijackErr), true)
return
}
defer conn.Close()
copyHeader(rw.Header(), res.Header)
res.Header = rw.Header()
res.Body = nil // so res.Write only writes the headers; we have res.Body in backConn above
if err := res.Write(brw); err != nil {
//nolint:errorlint
p.errorHandler(rw, req, fmt.Errorf("response write: %s", err), true)
return
}
if err := brw.Flush(); err != nil {
//nolint:errorlint
p.errorHandler(rw, req, fmt.Errorf("response flush: %s", err), true)
return
}
bdp := U.NewBidirectionalPipe(req.Context(), conn, backConn)
//nolint:errcheck
bdp.Start()
}
func IsPrint(s string) bool {
for _, r := range s {
if r < ' ' || r > '~' {
return false
}
}
return true
}

View File

@@ -1,49 +0,0 @@
package reverseproxy
import (
"io"
"net/http"
"net/url"
"strings"
"testing"
nettypes "github.com/yusing/godoxy/internal/net/types"
)
type noopTransport struct{}
func (t noopTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return &http.Response{
StatusCode: http.StatusOK,
Body: io.NopCloser(strings.NewReader("Hello, world!")),
Request: req,
ContentLength: int64(len("Hello, world!")),
Header: http.Header{},
}, nil
}
type noopResponseWriter struct{}
func (w noopResponseWriter) Header() http.Header {
return http.Header{}
}
func (w noopResponseWriter) Write(b []byte) (int, error) {
return len(b), nil
}
func (w noopResponseWriter) WriteHeader(statusCode int) {
}
func BenchmarkReverseProxy(b *testing.B) {
var w noopResponseWriter
var req = http.Request{
Method: "GET",
URL: &url.URL{Scheme: "http", Host: "test"},
Body: io.NopCloser(strings.NewReader("Hello, world!")),
}
proxy := NewReverseProxy("test", nettypes.MustParseURL("http://localhost:8080"), noopTransport{})
for b.Loop() {
proxy.ServeHTTP(w, &req)
}
}

View File

@@ -14,7 +14,6 @@ import (
"github.com/yusing/godoxy/internal/route"
"github.com/yusing/godoxy/internal/serialization"
"github.com/yusing/godoxy/internal/types"
"github.com/yusing/godoxy/internal/utils/strutils"
"github.com/yusing/godoxy/internal/watcher"
)
@@ -157,7 +156,7 @@ func (p *DockerProvider) routesFromContainerLabels(container *types.Container) (
// check if it is an alias reference
switch alias[0] {
case aliasRefPrefix, aliasRefPrefixAlt:
index, err := strutils.Atoi(alias[1:])
index, err := strconv.Atoi(alias[1:])
if err != nil {
errs.Add(err)
break

View File

@@ -13,13 +13,13 @@ import (
gphttp "github.com/yusing/godoxy/internal/net/gphttp"
"github.com/yusing/godoxy/internal/net/gphttp/loadbalancer"
"github.com/yusing/godoxy/internal/net/gphttp/middleware"
"github.com/yusing/godoxy/internal/net/gphttp/reverseproxy"
nettypes "github.com/yusing/godoxy/internal/net/types"
"github.com/yusing/godoxy/internal/route/routes"
"github.com/yusing/godoxy/internal/task"
"github.com/yusing/godoxy/internal/types"
"github.com/yusing/godoxy/internal/watcher/health/monitor"
"github.com/yusing/godoxy/pkg"
"github.com/yusing/goutils/http/reverseproxy"
)
type ReveseProxyRoute struct {
@@ -59,7 +59,7 @@ func NewReverseProxyRoute(base *Route) (*ReveseProxyRoute, gperr.Error) {
}
service := base.Name()
rp := reverseproxy.NewReverseProxy(service, proxyURL, trans)
rp := reverseproxy.NewReverseProxy(service, &proxyURL.URL, trans)
if len(base.Middlewares) > 0 {
err := middleware.PatchReverseProxy(rp, base.Middlewares)

View File

@@ -19,7 +19,7 @@ import (
"github.com/yusing/godoxy/internal/proxmox"
"github.com/yusing/godoxy/internal/task"
"github.com/yusing/godoxy/internal/types"
"github.com/yusing/godoxy/internal/utils/strutils"
strutils "github.com/yusing/goutils/strings"
"github.com/yusing/godoxy/internal/common"
config "github.com/yusing/godoxy/internal/config/types"

View File

@@ -8,9 +8,9 @@ import (
"github.com/yusing/godoxy/internal/gperr"
gphttp "github.com/yusing/godoxy/internal/net/gphttp"
"github.com/yusing/godoxy/internal/net/gphttp/reverseproxy"
nettypes "github.com/yusing/godoxy/internal/net/types"
"github.com/yusing/godoxy/internal/utils/strutils"
"github.com/yusing/goutils/http/reverseproxy"
strutils "github.com/yusing/goutils/strings"
)
type (
@@ -164,7 +164,7 @@ var commands = map[string]struct {
if target.Scheme == "" {
target.Scheme = "http"
}
rp := reverseproxy.NewReverseProxy("", target, gphttp.NewTransport())
rp := reverseproxy.NewReverseProxy("", &target.URL, gphttp.NewTransport())
return ReturningCommand(rp.ServeHTTP)
},
},

View File

@@ -10,7 +10,7 @@ import (
"github.com/yusing/godoxy/internal/gperr"
nettypes "github.com/yusing/godoxy/internal/net/types"
"github.com/yusing/godoxy/internal/route/routes"
"github.com/yusing/godoxy/internal/utils/strutils"
strutils "github.com/yusing/goutils/strings"
)
type RuleOn struct {

View File

@@ -8,7 +8,7 @@ import (
"github.com/rs/zerolog"
config "github.com/yusing/godoxy/internal/config/types"
nettypes "github.com/yusing/godoxy/internal/net/types"
"github.com/yusing/godoxy/internal/utils"
ioutils "github.com/yusing/goutils/io"
"go.uber.org/atomic"
)
@@ -153,7 +153,7 @@ func (s *TCPTCPStream) handle(ctx context.Context, conn net.Conn) {
}
}
pipe := utils.NewBidirectionalPipe(ctx, src, dst)
pipe := ioutils.NewBidirectionalPipe(ctx, src, dst)
if err := pipe.Start(); err != nil && !s.closed.Load() {
logErr(s, err, "error in bidirectional pipe")
}

View File

@@ -12,7 +12,7 @@ import (
"github.com/rs/zerolog"
config "github.com/yusing/godoxy/internal/config/types"
nettypes "github.com/yusing/godoxy/internal/net/types"
"github.com/yusing/godoxy/internal/utils/synk"
"github.com/yusing/goutils/synk"
"go.uber.org/atomic"
)

View File

@@ -4,7 +4,7 @@ import (
"strconv"
"github.com/yusing/godoxy/internal/gperr"
"github.com/yusing/godoxy/internal/utils/strutils"
strutils "github.com/yusing/goutils/strings"
)
type Port struct {

View File

@@ -15,7 +15,7 @@ import (
"github.com/puzpuzpuz/xsync/v4"
"github.com/yusing/godoxy/internal/gperr"
"github.com/yusing/godoxy/internal/utils"
"github.com/yusing/godoxy/internal/utils/strutils"
strutils "github.com/yusing/goutils/strings"
)
type SerializedObject = map[string]any

View File

@@ -8,7 +8,7 @@ import (
"time"
"github.com/yusing/godoxy/internal/task"
"github.com/yusing/godoxy/internal/utils/strutils"
strutils "github.com/yusing/goutils/strings"
)
type (

View File

@@ -4,7 +4,7 @@ import (
"net/http"
nettypes "github.com/yusing/godoxy/internal/net/types"
"github.com/yusing/godoxy/internal/utils/strutils"
strutils "github.com/yusing/goutils/strings"
)
type (

View File

@@ -5,10 +5,10 @@ import (
"github.com/yusing/godoxy/agent/pkg/agent"
"github.com/yusing/godoxy/internal/homepage"
"github.com/yusing/godoxy/internal/net/gphttp/reverseproxy"
nettypes "github.com/yusing/godoxy/internal/net/types"
"github.com/yusing/godoxy/internal/task"
"github.com/yusing/godoxy/internal/utils/pool"
"github.com/yusing/goutils/http/reverseproxy"
)
type (

View File

@@ -1,210 +0,0 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Modified from bufio.Writer by yusing <yusing@6uo.me>.
package utils
import (
"io"
"unicode/utf8"
)
// buffered output
// BufferedWriter implements buffering for an [io.BufferedWriter] object.
// If an error occurs writing to a [BufferedWriter], no more data will be
// accepted and all subsequent writes, and [BufferedWriter.Flush], will return the error.
// After all data has been written, the client should call the
// [BufferedWriter.Flush] method to guarantee all data has been forwarded to
// the underlying [io.BufferedWriter].
type BufferedWriter struct {
err error
buf []byte
n int
wr io.Writer
}
// NewBufferedWriter returns a new [BufferedWriter] whose buffer has at least the specified
// size. If the argument io.Writer is already a [BufferedWriter] with large enough
// size, it returns the underlying [BufferedWriter].
func NewBufferedWriter(w io.Writer, size int) *BufferedWriter {
// Is it already a Writer?
b, ok := w.(*BufferedWriter)
if ok && len(b.buf) >= size {
return b
}
return &BufferedWriter{
buf: bytesPool.GetSized(size),
wr: w,
}
}
// Size returns the size of the underlying buffer in bytes.
func (b *BufferedWriter) Size() int { return len(b.buf) }
func (b *BufferedWriter) Resize(size int) error {
err := b.Flush()
if err != nil {
return err
}
if cap(b.buf) >= size {
b.buf = b.buf[:size]
} else {
b.Release()
b.buf = bytesPool.GetSized(size)
}
b.err = nil
b.n = 0
return nil
}
func (b *BufferedWriter) Release() {
bytesPool.Put(b.buf)
}
// Flush writes any buffered data to the underlying [io.Writer].
func (b *BufferedWriter) Flush() error {
if b.err != nil {
return b.err
}
if b.n == 0 {
return nil
}
n, err := b.wr.Write(b.buf[0:b.n])
if n < b.n && err == nil {
err = io.ErrShortWrite
}
if err != nil {
if n > 0 && n < b.n {
copy(b.buf[0:b.n-n], b.buf[n:b.n])
}
b.n -= n
b.err = err
return err
}
b.n = 0
return nil
}
// Available returns how many bytes are unused in the buffer.
func (b *BufferedWriter) Available() int { return len(b.buf) - b.n }
// AvailableBuffer returns an empty buffer with b.Available() capacity.
// This buffer is intended to be appended to and
// passed to an immediately succeeding [BufferedWriter.Write] call.
// The buffer is only valid until the next write operation on b.
func (b *BufferedWriter) AvailableBuffer() []byte {
return b.buf[b.n:][:0]
}
// Buffered returns the number of bytes that have been written into the current buffer.
func (b *BufferedWriter) Buffered() int { return b.n }
// Write writes the contents of p into the buffer.
// It returns the number of bytes written.
// If nn < len(p), it also returns an error explaining
// why the write is short.
func (b *BufferedWriter) Write(p []byte) (nn int, err error) {
for len(p) > b.Available() && b.err == nil {
var n int
if b.Buffered() == 0 {
// Large write, empty buffer.
// Write directly from p to avoid copy.
n, b.err = b.wr.Write(p)
} else {
n = copy(b.buf[b.n:], p)
b.n += n
b.Flush()
}
nn += n
p = p[n:]
}
if b.err != nil {
return nn, b.err
}
n := copy(b.buf[b.n:], p)
b.n += n
nn += n
return nn, nil
}
// WriteByte writes a single byte.
func (b *BufferedWriter) WriteByte(c byte) error {
if b.err != nil {
return b.err
}
if b.Available() <= 0 && b.Flush() != nil {
return b.err
}
b.buf[b.n] = c
b.n++
return nil
}
// WriteRune writes a single Unicode code point, returning
// the number of bytes written and any error.
func (b *BufferedWriter) WriteRune(r rune) (size int, err error) {
// Compare as uint32 to correctly handle negative runes.
if uint32(r) < utf8.RuneSelf {
err = b.WriteByte(byte(r))
if err != nil {
return 0, err
}
return 1, nil
}
if b.err != nil {
return 0, b.err
}
n := b.Available()
if n < utf8.UTFMax {
if b.Flush(); b.err != nil {
return 0, b.err
}
n = b.Available()
if n < utf8.UTFMax {
// Can only happen if buffer is silly small.
return b.WriteString(string(r))
}
}
size = utf8.EncodeRune(b.buf[b.n:], r)
b.n += size
return size, nil
}
// WriteString writes a string.
// It returns the number of bytes written.
// If the count is less than len(s), it also returns an error explaining
// why the write is short.
func (b *BufferedWriter) WriteString(s string) (int, error) {
var sw io.StringWriter
tryStringWriter := true
nn := 0
for len(s) > b.Available() && b.err == nil {
var n int
if b.Buffered() == 0 && sw == nil && tryStringWriter {
// Check at most once whether b.wr is a StringWriter.
sw, tryStringWriter = b.wr.(io.StringWriter)
}
if b.Buffered() == 0 && tryStringWriter {
// Large write, empty buffer, and the underlying writer supports
// WriteString: forward the write to the underlying StringWriter.
// This avoids an extra copy.
n, b.err = sw.WriteString(s)
} else {
n = copy(b.buf[b.n:], s)
b.n += n
b.Flush()
}
nn += n
s = s[n:]
}
if b.err != nil {
return nn, b.err
}
n := copy(b.buf[b.n:], s)
b.n += n
nn += n
return nn, nil
}

View File

@@ -6,8 +6,8 @@ require (
github.com/puzpuzpuz/xsync/v4 v4.2.0
github.com/rs/zerolog v1.34.0
github.com/stretchr/testify v1.11.1
github.com/yusing/goutils v0.2.1
go.uber.org/atomic v1.11.0
golang.org/x/text v0.29.0
)
require (
@@ -16,5 +16,6 @@ require (
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
golang.org/x/sys v0.36.0 // indirect
golang.org/x/text v0.29.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

View File

@@ -19,6 +19,8 @@ github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY=
github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/yusing/goutils v0.2.1 h1:KjoCrNO0otthaPCZPfQY+5GKsqs5+J77CxP+TNHYa/Y=
github.com/yusing/goutils v0.2.1/go.mod h1:v6RZsMRdzcts4udSg0vqUIFvaD0OaUMPTwYJZ4XnQYo=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

View File

@@ -1,259 +0,0 @@
package utils
import (
"context"
"errors"
"io"
"net/http"
"sync"
"syscall"
"github.com/yusing/godoxy/internal/utils/synk"
)
// TODO: move to "utils/io".
type (
FileReader struct {
Path string
}
ContextReader struct {
ctx context.Context
io.Reader
}
ContextWriter struct {
ctx context.Context
io.Writer
}
Pipe struct {
r ContextReader
w ContextWriter
}
BidirectionalPipe struct {
pSrcDst *Pipe
pDstSrc *Pipe
}
HookCloser struct {
c io.ReadCloser
hook func()
}
)
func NewContextReader(ctx context.Context, r io.Reader) *ContextReader {
return &ContextReader{ctx: ctx, Reader: r}
}
func NewContextWriter(ctx context.Context, w io.Writer) *ContextWriter {
return &ContextWriter{ctx: ctx, Writer: w}
}
func (r *ContextReader) Read(p []byte) (int, error) {
select {
case <-r.ctx.Done():
return 0, r.ctx.Err()
default:
return r.Reader.Read(p)
}
}
func (w *ContextWriter) Write(p []byte) (int, error) {
select {
case <-w.ctx.Done():
return 0, w.ctx.Err()
default:
return w.Writer.Write(p)
}
}
func NewPipe(ctx context.Context, r io.ReadCloser, w io.WriteCloser) *Pipe {
return &Pipe{
r: ContextReader{ctx: ctx, Reader: r},
w: ContextWriter{ctx: ctx, Writer: w},
}
}
func (p *Pipe) Start() (err error) {
err = CopyClose(&p.w, &p.r, 0)
switch {
case
// NOTE: ignoring broken pipe and connection reset by peer
errors.Is(err, syscall.EPIPE),
errors.Is(err, syscall.ECONNRESET):
return nil
}
return err
}
func NewBidirectionalPipe(ctx context.Context, rw1 io.ReadWriteCloser, rw2 io.ReadWriteCloser) BidirectionalPipe {
return BidirectionalPipe{
pSrcDst: NewPipe(ctx, rw1, rw2),
pDstSrc: NewPipe(ctx, rw2, rw1),
}
}
func (p BidirectionalPipe) Start() error {
var wg sync.WaitGroup
var srcErr, dstErr error
wg.Go(func() {
srcErr = p.pSrcDst.Start()
})
wg.Go(func() {
dstErr = p.pDstSrc.Start()
})
wg.Wait()
return errors.Join(srcErr, dstErr)
}
type flushErrorInterface interface {
FlushError() error
}
type flusherWrapper struct {
rw http.Flusher
}
type rwUnwrapper interface {
Unwrap() http.ResponseWriter
}
func (f *flusherWrapper) FlushError() error {
f.rw.Flush()
return nil
}
func getHTTPFlusher(dst io.Writer) flushErrorInterface {
// pre-unwrap the flusher to prevent unwrap and check in every loop
if rw, ok := dst.(http.ResponseWriter); ok {
for {
switch t := rw.(type) {
case flushErrorInterface:
return t
case http.Flusher:
return &flusherWrapper{rw: t}
case rwUnwrapper:
rw = t.Unwrap()
default:
return nil
}
}
}
return nil
}
const copyBufSize = synk.SizedPoolThreshold
var bytesPool = synk.GetBytesPool()
// ReadAllBody reads the body of the response into a buffer and returns it and a function to release the buffer.
func ReadAllBody(resp *http.Response) (buf []byte, release func(), err error) {
if contentLength := resp.ContentLength; contentLength > 0 {
buf = bytesPool.GetSized(int(contentLength))
_, err = io.ReadFull(resp.Body, buf)
if err != nil {
bytesPool.Put(buf)
return nil, nil, err
}
return buf, func() { bytesPool.Put(buf) }, nil
}
buf, err = io.ReadAll(resp.Body)
if err != nil {
bytesPool.Put(buf)
return nil, nil, err
}
return buf, func() { bytesPool.Put(buf) }, nil
}
// NewHookCloser wraps a io.ReadCloser and calls the hook function when the closer is closed.
func NewHookCloser(c io.ReadCloser, hook func()) *HookCloser {
return &HookCloser{hook: hook, c: c}
}
// Close calls the hook function and closes the underlying reader
func (r *HookCloser) Close() error {
r.hook()
return r.c.Close()
}
// Read reads from the underlying reader.
func (r *HookCloser) Read(p []byte) (int, error) {
return r.c.Read(p)
}
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// This is a copy of io.Copy with context and HTTP flusher handling
// Author: yusing <yusing@6uo.me>.
func CopyClose(dst *ContextWriter, src *ContextReader, sizeHint int) (err error) {
size := copyBufSize
if l, ok := src.Reader.(*io.LimitedReader); ok {
if int64(size) > l.N {
if l.N < 1 {
size = 1
} else {
size = int(l.N)
}
}
} else if sizeHint > 0 {
size = min(size, sizeHint)
}
buf := bytesPool.GetSized(size)
defer bytesPool.Put(buf)
// close both as soon as one of them is done
wCloser, wCanClose := dst.Writer.(io.Closer)
rCloser, rCanClose := src.Reader.(io.Closer)
if wCanClose || rCanClose {
go func() {
select {
case <-src.ctx.Done():
case <-dst.ctx.Done():
}
if rCanClose {
defer rCloser.Close()
}
if wCanClose {
defer wCloser.Close()
}
}()
}
flusher := getHTTPFlusher(dst.Writer)
for {
nr, er := src.Reader.Read(buf)
if nr > 0 {
nw, ew := dst.Writer.Write(buf[0:nr])
if nw < 0 || nr < nw {
nw = 0
if ew == nil {
ew = errors.New("invalid write result")
}
}
if ew != nil {
err = ew
return
}
if nr != nw {
err = io.ErrShortWrite
return
}
if flusher != nil {
err = flusher.FlushError()
if err != nil {
return err
}
}
}
if er != nil {
if er != io.EOF {
err = er
}
return
}
}
}
func CopyCloseWithContext(ctx context.Context, dst io.Writer, src io.Reader, sizeHint int) (err error) {
return CopyClose(NewContextWriter(ctx, dst), NewContextReader(ctx, src), sizeHint)
}

View File

@@ -3,7 +3,7 @@ package utils
import (
"reflect"
"github.com/yusing/godoxy/internal/utils/strutils"
strutils "github.com/yusing/goutils/strings"
)
func NearestField(input string, s any) string {

View File

@@ -1,47 +0,0 @@
package ansi
import (
"regexp"
)
var ansiRegexp = regexp.MustCompile(`\x1b\[[0-9;]*m`)
const (
BrightRed = "\x1b[91m"
BrightGreen = "\x1b[92m"
BrightYellow = "\x1b[93m"
BrightCyan = "\x1b[96m"
BrightWhite = "\x1b[97m"
Bold = "\x1b[1m"
Reset = "\x1b[0m"
HighlightRed = BrightRed + Bold
HighlightGreen = BrightGreen + Bold
HighlightYellow = BrightYellow + Bold
HighlightCyan = BrightCyan + Bold
HighlightWhite = BrightWhite + Bold
)
func Error(s string) string {
return WithANSI(s, HighlightRed)
}
func Success(s string) string {
return WithANSI(s, HighlightGreen)
}
func Warning(s string) string {
return WithANSI(s, HighlightYellow)
}
func Info(s string) string {
return WithANSI(s, HighlightCyan)
}
func WithANSI(s string, ansi string) string {
return ansi + s + Reset
}
func StripANSI(s string) string {
return ansiRegexp.ReplaceAllString(s, "")
}

View File

@@ -1,11 +0,0 @@
package strutils
import "strings"
// IsValidFilename checks if a filename is safe and doesn't contain path traversal attempts
// Returns true if the filename is valid, false otherwise
func IsValidFilename(filename string) bool {
return !strings.Contains(filename, "/") &&
!strings.Contains(filename, "\\") &&
!strings.Contains(filename, "..")
}

View File

@@ -1,221 +0,0 @@
package strutils
import (
"fmt"
"math"
"strconv"
"time"
)
// AppendDuration appends a duration to a buffer with the following format:
// - 1 ns
// - 1 ms
// - 1 seconds
// - 1 minutes and 1 seconds
// - 1 hours, 1 minutes and 1 seconds
// - 1 days, 1 hours and 1 minutes (ignore seconds if days >= 1)
func AppendDuration(d time.Duration, buf []byte) []byte {
if d < 0 {
buf = append(buf, '-')
d = -d
}
if d == 0 {
return append(buf, []byte("0 Seconds")...)
}
switch {
case d < time.Millisecond:
buf = strconv.AppendInt(buf, d.Nanoseconds(), 10)
buf = append(buf, []byte(" ns")...)
return buf
case d < time.Second:
buf = strconv.AppendInt(buf, d.Milliseconds(), 10)
buf = append(buf, []byte(" ms")...)
return buf
}
// Get total seconds from duration
totalSeconds := int64(d.Seconds())
// Calculate days, hours, minutes, and seconds
days := totalSeconds / (24 * 3600)
hours := (totalSeconds % (24 * 3600)) / 3600
minutes := (totalSeconds % 3600) / 60
seconds := totalSeconds % 60
idxPartBeg := 0
if days > 0 {
buf = strconv.AppendInt(buf, days, 10)
buf = fmt.Appendf(buf, " day%s, ", Pluralize(days))
}
if hours > 0 {
idxPartBeg = len(buf) - 2
buf = strconv.AppendInt(buf, hours, 10)
buf = fmt.Appendf(buf, " hour%s, ", Pluralize(hours))
}
if minutes > 0 {
idxPartBeg = len(buf) - 2
buf = strconv.AppendInt(buf, minutes, 10)
buf = fmt.Appendf(buf, " minute%s, ", Pluralize(minutes))
}
if seconds > 0 && totalSeconds < 3600 {
idxPartBeg = len(buf) - 2
buf = strconv.AppendInt(buf, seconds, 10)
buf = fmt.Appendf(buf, " second%s, ", Pluralize(seconds))
}
// remove last comma and space
buf = buf[:len(buf)-2]
if idxPartBeg > 0 && idxPartBeg < len(buf) {
// replace last part ', ' with ' and ' in-place, alloc-free
// ', ' is 2 bytes, ' and ' is 5 bytes, so we need to make room for 3 more bytes
tailLen := len(buf) - (idxPartBeg + 2)
buf = append(buf, "000"...) // append 3 bytes for ' and '
copy(buf[idxPartBeg+5:], buf[idxPartBeg+2:idxPartBeg+2+tailLen]) // shift tail right by 3
copy(buf[idxPartBeg:], " and ") // overwrite ', ' with ' and '
}
return buf
}
func FormatDuration(d time.Duration) string {
return string(AppendDuration(d, nil))
}
func FormatLastSeen(t time.Time) string {
if t.IsZero() {
return "never"
}
return FormatTime(t)
}
func appendRound(f float64, buf []byte) []byte {
return strconv.AppendInt(buf, int64(math.Round(f)), 10)
}
func appendFloat(f float64, buf []byte) []byte {
f = math.Round(f*100) / 100
if f == 0 {
return buf
}
return strconv.AppendFloat(buf, f, 'f', -1, 64)
}
func AppendTime(t time.Time, buf []byte) []byte {
if t.IsZero() {
return append(buf, []byte("never")...)
}
return AppendTimeWithReference(t, time.Now(), buf)
}
func FormatTime(t time.Time) string {
return string(AppendTime(t, nil))
}
func FormatUnixTime(t int64) string {
return FormatTime(time.Unix(t, 0))
}
func FormatTimeWithReference(t, ref time.Time) string {
return string(AppendTimeWithReference(t, ref, nil))
}
func AppendTimeWithReference(t, ref time.Time, buf []byte) []byte {
if t.IsZero() {
return append(buf, []byte("never")...)
}
diff := t.Sub(ref)
absDiff := diff.Abs()
switch {
case absDiff < time.Second:
return append(buf, []byte("now")...)
case absDiff < 3*time.Second:
if diff < 0 {
return append(buf, []byte("just now")...)
}
fallthrough
case absDiff < 60*time.Second:
if diff < 0 {
buf = appendRound(absDiff.Seconds(), buf)
buf = append(buf, []byte(" seconds ago")...)
} else {
buf = append(buf, []byte("in ")...)
buf = appendRound(absDiff.Seconds(), buf)
buf = append(buf, []byte(" seconds")...)
}
return buf
case absDiff < 60*time.Minute:
if diff < 0 {
buf = appendRound(absDiff.Minutes(), buf)
buf = append(buf, []byte(" minutes ago")...)
} else {
buf = append(buf, []byte("in ")...)
buf = appendRound(absDiff.Minutes(), buf)
buf = append(buf, []byte(" minutes")...)
}
return buf
case absDiff < 24*time.Hour:
if diff < 0 {
buf = appendRound(absDiff.Hours(), buf)
buf = append(buf, []byte(" hours ago")...)
} else {
buf = append(buf, []byte("in ")...)
buf = appendRound(absDiff.Hours(), buf)
buf = append(buf, []byte(" hours")...)
}
return buf
case t.Year() == ref.Year():
return t.AppendFormat(buf, "01-02 15:04:05")
default:
return t.AppendFormat(buf, "2006-01-02 15:04:05")
}
}
func FormatByteSize[T ~int | ~uint | ~int64 | ~uint64 | ~float64](size T) string {
return string(AppendByteSize(size, nil))
}
func AppendByteSize[T ~int | ~uint | ~int64 | ~uint64 | ~float64](size T, buf []byte) []byte {
const (
_ = (1 << (10 * iota))
kb
mb
gb
tb
pb
)
switch {
case size < kb:
switch any(size).(type) {
case int, int64:
buf = strconv.AppendInt(buf, int64(size), 10)
case uint, uint64:
buf = strconv.AppendUint(buf, uint64(size), 10)
case float64:
buf = appendFloat(float64(size), buf)
}
buf = append(buf, []byte(" B")...)
case size < mb:
buf = appendFloat(float64(size)/kb, buf)
buf = append(buf, []byte(" KiB")...)
case size < gb:
buf = appendFloat(float64(size)/mb, buf)
buf = append(buf, []byte(" MiB")...)
case size < tb:
buf = appendFloat(float64(size)/gb, buf)
buf = append(buf, []byte(" GiB")...)
case size < pb:
buf = appendFloat(float64(size/gb)/kb, buf)
buf = append(buf, []byte(" TiB")...)
default:
buf = appendFloat(float64(size/tb)/kb, buf)
buf = append(buf, []byte(" PiB")...)
}
return buf
}
func Pluralize(n int64) string {
if n > 1 {
return "s"
}
return ""
}

View File

@@ -1,302 +0,0 @@
package strutils_test
import (
"testing"
"time"
"github.com/stretchr/testify/require"
. "github.com/yusing/godoxy/internal/utils/strutils"
)
func mustParseTime(t *testing.T, layout, value string) time.Time {
t.Helper()
time, err := time.Parse(layout, value)
if err != nil {
t.Fatalf("failed to parse time: %s", err)
}
return time
}
func TestFormatTime(t *testing.T) {
now := mustParseTime(t, time.RFC3339, "2021-06-15T12:30:30Z")
tests := []struct {
name string
time time.Time
expected string
expectedLength int
}{
{
name: "now",
time: now.Add(100 * time.Millisecond),
expected: "now",
},
{
name: "just now (past within 3 seconds)",
time: now.Add(-1 * time.Second),
expected: "just now",
},
{
name: "seconds ago",
time: now.Add(-10 * time.Second),
expected: "10 seconds ago",
},
{
name: "in seconds",
time: now.Add(10 * time.Second),
expected: "in 10 seconds",
},
{
name: "minutes ago",
time: now.Add(-10 * time.Minute),
expected: "10 minutes ago",
},
{
name: "in minutes",
time: now.Add(10 * time.Minute),
expected: "in 10 minutes",
},
{
name: "hours ago",
time: now.Add(-10 * time.Hour),
expected: "10 hours ago",
},
{
name: "in hours",
time: now.Add(10 * time.Hour),
expected: "in 10 hours",
},
{
name: "different day",
time: now.Add(-25 * time.Hour),
expectedLength: len("01-01 15:04:05"),
},
{
name: "same year but different month",
time: now.Add(-30 * 24 * time.Hour),
expectedLength: len("01-01 15:04:05"),
},
{
name: "different year",
time: time.Date(now.Year()-1, 1, 1, 10, 20, 30, 0, now.Location()),
expected: time.Date(now.Year()-1, 1, 1, 10, 20, 30, 0, now.Location()).Format("2006-01-02 15:04:05"),
},
{
name: "zero time",
time: time.Time{},
expected: "never",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := FormatTimeWithReference(tt.time, now)
if tt.expectedLength > 0 {
require.Len(t, result, tt.expectedLength)
} else {
require.Equal(t, tt.expected, result)
}
})
}
}
func TestFormatDuration(t *testing.T) {
tests := []struct {
name string
duration time.Duration
expected string
}{
{
name: "zero duration",
duration: 0,
expected: "0 Seconds",
},
{
name: "seconds only",
duration: 45 * time.Second,
expected: "45 seconds",
},
{
name: "one second",
duration: 1 * time.Second,
expected: "1 second",
},
{
name: "minutes only",
duration: 5 * time.Minute,
expected: "5 minutes",
},
{
name: "one minute",
duration: 1 * time.Minute,
expected: "1 minute",
},
{
name: "hours only",
duration: 3 * time.Hour,
expected: "3 hours",
},
{
name: "one hour",
duration: 1 * time.Hour,
expected: "1 hour",
},
{
name: "days only",
duration: 2 * 24 * time.Hour,
expected: "2 days",
},
{
name: "one day",
duration: 24 * time.Hour,
expected: "1 day",
},
{
name: "complex duration",
duration: 2*24*time.Hour + 3*time.Hour + 45*time.Minute + 15*time.Second,
expected: "2 days, 3 hours and 45 minutes",
},
{
name: "hours and minutes",
duration: 2*time.Hour + 30*time.Minute,
expected: "2 hours and 30 minutes",
},
{
name: "days and hours",
duration: 1*24*time.Hour + 12*time.Hour,
expected: "1 day and 12 hours",
},
{
name: "days and hours and minutes",
duration: 1*24*time.Hour + 12*time.Hour + 30*time.Minute,
expected: "1 day, 12 hours and 30 minutes",
},
{
name: "days and hours and minutes and seconds (ignore seconds)",
duration: 1*24*time.Hour + 12*time.Hour + 30*time.Minute + 15*time.Second,
expected: "1 day, 12 hours and 30 minutes",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := FormatDuration(tt.duration)
require.Equal(t, tt.expected, result)
})
}
}
func TestFormatLastSeen(t *testing.T) {
now := time.Now()
tests := []struct {
name string
time time.Time
expected string
}{
{
name: "zero time",
time: time.Time{},
expected: "never",
},
{
name: "non-zero time",
time: now.Add(-10 * time.Minute),
// The actual result will be handled by FormatTime, which is tested separately
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := FormatLastSeen(tt.time)
if tt.name == "zero time" {
require.Equal(t, tt.expected, result)
} else if result == "never" { // Just make sure it's not "never", the actual formatting is tested in TestFormatTime
t.Errorf("Expected non-zero time to not return 'never', got %s", result)
}
})
}
}
func TestFormatByteSize(t *testing.T) {
tests := []struct {
name string
size int64
expected string
}{
{
name: "zero size",
size: 0,
expected: "0 B",
},
{
name: "one byte",
size: 1,
expected: "1 B",
},
{
name: "bytes (less than 1 KiB)",
size: 1023,
expected: "1023 B",
},
{
name: "1 KiB",
size: 1024,
expected: "1 KiB",
},
{
name: "KiB (less than 1 MiB)",
size: 1024 * 1023,
expected: "1023 KiB",
},
{
name: "1 MiB",
size: 1024 * 1024,
expected: "1 MiB",
},
{
name: "MiB (less than 1 GiB)",
size: 1024 * 1024 * 1023,
expected: "1023 MiB",
},
{
name: "1 GiB",
size: 1024 * 1024 * 1024,
expected: "1 GiB",
},
{
name: "GiB (less than 1 TiB)",
size: 1024 * 1024 * 1024 * 1023,
expected: "1023 GiB",
},
{
name: "1 TiB",
size: 1024 * 1024 * 1024 * 1024,
expected: "1 TiB",
},
{
name: "TiB (less than 1 PiB)",
size: 1024 * 1024 * 1024 * 1024 * 1023,
expected: "1023 TiB",
},
{
name: "1 PiB",
size: 1024 * 1024 * 1024 * 1024 * 1024,
expected: "1 PiB",
},
{
name: "PiB (large number)",
size: 1024 * 1024 * 1024 * 1024 * 1024 * 1023,
expected: "1023 PiB",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := FormatByteSize(tt.size)
require.Equal(t, tt.expected, result)
})
}
}

View File

@@ -1,26 +0,0 @@
package strutils
import (
"reflect"
)
type Parser interface {
Parse(value string) error
}
func Parse[T Parser](from string) (t T, err error) {
tt := reflect.TypeOf(t)
if tt.Kind() == reflect.Ptr {
t = reflect.New(tt.Elem()).Interface().(T)
}
err = t.Parse(from)
return t, err
}
func MustParse[T Parser](from string) T {
t, err := Parse[T](from)
if err != nil {
panic("must failed: " + err.Error())
}
return t
}

View File

@@ -1,81 +0,0 @@
package strutils
import (
"math"
"strings"
)
// SplitRune is like strings.Split but takes a rune as separator.
func SplitRune(s string, sep rune) []string {
if sep == 0 {
return strings.Split(s, "")
}
n := strings.Count(s, string(sep)) + 1
if n > len(s)+1 {
n = len(s) + 1
}
a := make([]string, n)
n--
i := 0
for i < n {
m := strings.IndexRune(s, sep)
if m < 0 {
break
}
a[i] = s[:m]
s = s[m+1:]
i++
}
a[i] = s
return a[:i+1]
}
// SplitComma is a wrapper around SplitRune(s, ',').
func SplitComma(s string) []string {
return SplitRune(s, ',')
}
// SplitLine is a wrapper around SplitRune(s, '\n').
func SplitLine(s string) []string {
return SplitRune(s, '\n')
}
// SplitSpace is a wrapper around SplitRune(s, ' ').
func SplitSpace(s string) []string {
return SplitRune(s, ' ')
}
// JoinRune is like strings.Join but takes a rune as separator.
func JoinRune(elems []string, sep rune) string {
switch len(elems) {
case 0:
return ""
case 1:
return elems[0]
}
if sep == 0 {
return strings.Join(elems, "")
}
var n int
for _, elem := range elems {
if len(elem) > math.MaxInt-n {
panic("strings: Join output length overflow")
}
n += len(elem)
}
var b strings.Builder
b.Grow(n)
b.WriteString(elems[0])
for _, s := range elems[1:] {
b.WriteRune(sep)
b.WriteString(s)
}
return b.String()
}
// JoinLines is a wrapper around JoinRune(elems, '\n').
func JoinLines(elems []string) string {
return JoinRune(elems, '\n')
}

View File

@@ -1,62 +0,0 @@
package strutils_test
import (
"strings"
"testing"
"github.com/stretchr/testify/require"
. "github.com/yusing/godoxy/internal/utils/strutils"
)
var alphaNumeric = func() string {
var s strings.Builder
for i := range 'z' - 'a' + 1 {
s.WriteRune('a' + i)
s.WriteRune('A' + i)
s.WriteRune(',')
}
for i := range '9' - '0' + 1 {
s.WriteRune('0' + i)
s.WriteRune(',')
}
return s.String()
}()
func TestSplit(t *testing.T) {
tests := map[string]rune{
"": 0,
"1": '1',
",": ',',
}
for sep, rsep := range tests {
t.Run(sep, func(t *testing.T) {
expected := strings.Split(alphaNumeric, sep)
require.Equal(t, expected, SplitRune(alphaNumeric, rsep))
require.Equal(t, alphaNumeric, JoinRune(expected, rsep))
})
}
}
func BenchmarkSplitRune(b *testing.B) {
for range b.N {
SplitRune(alphaNumeric, ',')
}
}
func BenchmarkSplitRuneStdlib(b *testing.B) {
for range b.N {
strings.Split(alphaNumeric, ",")
}
}
func BenchmarkJoinRune(b *testing.B) {
for range b.N {
JoinRune(SplitRune(alphaNumeric, ','), ',')
}
}
func BenchmarkJoinRuneStdlib(b *testing.B) {
for range b.N {
strings.Join(SplitRune(alphaNumeric, ','), ",")
}
}

View File

@@ -1,7 +0,0 @@
package strutils
import (
"strconv"
)
var Atoi = strconv.Atoi

View File

@@ -1,98 +0,0 @@
package strutils
import (
"strings"
"golang.org/x/text/cases"
"golang.org/x/text/language"
)
// CommaSeperatedList returns a list of strings split by commas,
// then trim spaces from each element.
func CommaSeperatedList(s string) []string {
if s == "" {
return []string{}
}
res := SplitComma(s)
for i, part := range res {
res[i] = strings.TrimSpace(part)
}
return res
}
var caseTitle = cases.Title(language.AmericanEnglish)
func Title(s string) string {
return caseTitle.String(s)
}
func ContainsFold(s, substr string) bool {
return IndexFold(s, substr) >= 0
}
func IndexFold(s, substr string) int {
return strings.Index(strings.ToLower(s), strings.ToLower(substr))
}
func ToLowerNoSnake(s string) string {
var buf strings.Builder
for _, r := range s {
if r == '_' {
continue
}
if r >= 'A' && r <= 'Z' {
r += 'a' - 'A'
}
buf.WriteRune(r)
}
return buf.String()
}
//nolint:intrange
func LevenshteinDistance(a, b string) int {
if a == b {
return 0
}
if len(a) == 0 {
return len(b)
}
if len(b) == 0 {
return len(a)
}
v0 := make([]int, len(b)+1)
v1 := make([]int, len(b)+1)
for i := 0; i <= len(b); i++ {
v0[i] = i
}
for i := 0; i < len(a); i++ {
v1[0] = i + 1
for j := 0; j < len(b); j++ {
cost := 0
if a[i] != b[j] {
cost = 1
}
v1[j+1] = min3(v1[j]+1, v0[j+1]+1, v0[j]+cost)
}
for j := 0; j <= len(b); j++ {
v0[j] = v1[j]
}
}
return v1[len(b)]
}
func min3(a, b, c int) int {
if a < b && a < c {
return a
}
if b < a && b < c {
return b
}
return c
}

View File

@@ -1,26 +0,0 @@
package strutils
import (
"path"
"strings"
)
// SanitizeURI sanitizes a URI reference to ensure it is safe
// It disallows URLs beginning with // or /\ as absolute URLs,
// cleans the URL path to remove any .. or . path elements,
// and ensures the URL starts with a / if it doesn't already
func SanitizeURI(uri string) string {
if uri == "" {
return "/"
}
if strings.HasPrefix(uri, "http://") || strings.HasPrefix(uri, "https://") {
return uri
}
if uri[0] != '/' {
uri = "/" + uri
}
if len(uri) > 1 && uri[0] == '/' && uri[1] != '/' && uri[1] != '\\' {
return path.Clean(uri)
}
return "/"
}

View File

@@ -1,63 +0,0 @@
package strutils
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestSanitizeURI(t *testing.T) {
tests := []struct {
name string
input string
expected string
}{
{
name: "empty string",
input: "",
expected: "/",
},
{
name: "single slash",
input: "/",
expected: "/",
},
{
name: "normal path",
input: "/path/to/resource",
expected: "/path/to/resource",
},
{
name: "path without leading slash",
input: "path/to/resource",
expected: "/path/to/resource",
},
{
name: "path with dot segments",
input: "/path/./to/../resource",
expected: "/path/resource",
},
{
name: "double slash prefix",
input: "//path/to/resource",
expected: "/",
},
{
name: "backslash prefix",
input: "/\\path/to/resource",
expected: "/",
},
{
name: "path with multiple slashes",
input: "/path//to///resource",
expected: "/path/to/resource",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := SanitizeURI(tt.input)
require.Equal(t, tt.expected, result)
})
}
}

View File

@@ -1,238 +0,0 @@
package synk
import (
"sync/atomic"
"unsafe"
"weak"
)
type weakBuf = weak.Pointer[[]byte]
func makeWeak(b *[]byte) weakBuf {
return weak.Make(b)
}
func getBufFromWeak(w weakBuf) []byte {
ptr := w.Value()
if ptr != nil {
return *ptr
}
return nil
}
type BytesPool struct {
sizedPool chan weakBuf
unsizedPool chan weakBuf
initSize int
}
type BytesPoolWithMemory struct {
maxAllocatedSize atomic.Uint32
numShouldShrink atomic.Int32
pool chan weakBuf
}
type sliceInternal struct {
ptr unsafe.Pointer
len int
cap int
}
func sliceStruct(b *[]byte) *sliceInternal {
return (*sliceInternal)(unsafe.Pointer(b))
}
func underlyingPtr(b []byte) unsafe.Pointer {
return sliceStruct(&b).ptr
}
func setCap(b *[]byte, cap int) {
sliceStruct(b).cap = cap
}
func setLen(b *[]byte, len int) {
sliceStruct(b).len = len
}
const (
kb = 1024
mb = 1024 * kb
)
const (
InPoolLimit = 32 * mb
UnsizedAvg = 4 * kb
SizedPoolThreshold = 256 * kb
DropThreshold = 4 * mb
SizedPoolSize = InPoolLimit * 8 / 10 / SizedPoolThreshold
UnsizedPoolSize = InPoolLimit * 2 / 10 / UnsizedAvg
ShouldShrinkThreshold = 10
)
var bytesPool = &BytesPool{
sizedPool: make(chan weakBuf, SizedPoolSize),
unsizedPool: make(chan weakBuf, UnsizedPoolSize),
initSize: UnsizedAvg,
}
var bytesPoolWithMemory = make(chan weakBuf, UnsizedPoolSize)
func GetBytesPool() *BytesPool {
return bytesPool
}
func GetBytesPoolWithUniqueMemory() *BytesPoolWithMemory {
return &BytesPoolWithMemory{
pool: bytesPoolWithMemory,
}
}
func (p *BytesPool) Get() []byte {
for {
select {
case bWeak := <-p.unsizedPool:
bPtr := getBufFromWeak(bWeak)
if bPtr == nil {
continue
}
addReused(cap(bPtr))
return bPtr
default:
addNonPooled(p.initSize)
return make([]byte, 0, p.initSize)
}
}
}
func (p *BytesPoolWithMemory) Get() []byte {
for {
size := int(p.maxAllocatedSize.Load())
select {
case bWeak := <-p.pool:
bPtr := getBufFromWeak(bWeak)
if bPtr == nil {
continue
}
addReused(cap(bPtr))
return bPtr
default:
addNonPooled(size)
return make([]byte, 0, size)
}
}
}
func (p *BytesPool) GetSized(size int) []byte {
for {
select {
case bWeak := <-p.sizedPool:
b := getBufFromWeak(bWeak)
if b == nil {
continue
}
capB := cap(b)
remainingSize := capB - size
if remainingSize == 0 {
addReused(capB)
return b[:size]
}
if remainingSize > 0 { // capB > size (buffer larger than requested)
addReused(size)
p.Put(b[size:capB])
// return the first part and limit the capacity to the requested size
ret := b[:size]
setLen(&ret, size)
setCap(&ret, size)
return ret
}
// size is not enough
select {
case p.sizedPool <- bWeak:
default:
addDropped(cap(b))
// just drop it
}
default:
}
addNonPooled(size)
return make([]byte, size)
}
}
func (p *BytesPool) Put(b []byte) {
size := cap(b)
if size > DropThreshold {
addDropped(size)
return
}
b = b[:0]
if size >= SizedPoolThreshold {
p.put(size, makeWeak(&b), p.sizedPool)
} else {
p.put(size, makeWeak(&b), p.unsizedPool)
}
}
func (p *BytesPoolWithMemory) Put(b []byte) {
capB := uint32(cap(b))
for {
current := p.maxAllocatedSize.Load()
if capB < current {
// Potential shrink case
if p.numShouldShrink.Add(1) > ShouldShrinkThreshold {
if p.maxAllocatedSize.CompareAndSwap(current, capB) {
p.numShouldShrink.Store(0) // reset counter
break
}
p.numShouldShrink.Add(-1) // undo if CAS failed
}
break
} else if capB > current {
// Growing case
if p.maxAllocatedSize.CompareAndSwap(current, capB) {
break
}
// retry if CAS failed
} else {
// equal case - no change needed
break
}
}
if capB > DropThreshold {
addDropped(int(capB))
return
}
b = b[:0]
w := makeWeak(&b)
select {
case p.pool <- w:
default:
addDropped(int(capB))
// just drop it
}
}
//go:inline
func (p *BytesPool) put(size int, w weakBuf, pool chan weakBuf) {
select {
case pool <- w:
default:
addDropped(size)
// just drop it
}
}
func init() {
initPoolStats()
}

View File

@@ -1,69 +0,0 @@
package synk
import (
"slices"
"testing"
)
var sizes = []int{1024, 4096, 16384, 65536, 32 * 1024, 128 * 1024, 512 * 1024, 1024 * 1024, 2 * 1024 * 1024}
func BenchmarkBytesPool_GetSmall(b *testing.B) {
for b.Loop() {
bytesPool.Put(bytesPool.GetSized(1024))
}
}
func BenchmarkBytesPool_MakeSmall(b *testing.B) {
for b.Loop() {
_ = make([]byte, 1024)
}
}
func BenchmarkBytesPool_GetLarge(b *testing.B) {
for b.Loop() {
buf := bytesPool.GetSized(DropThreshold / 2)
buf[0] = 1
bytesPool.Put(buf)
}
}
func BenchmarkBytesPool_GetLargeUnsized(b *testing.B) {
for b.Loop() {
buf := slices.Grow(bytesPool.Get(), DropThreshold/2)
buf = append(buf, 1)
bytesPool.Put(buf)
}
}
func BenchmarkBytesPool_MakeLarge(b *testing.B) {
for b.Loop() {
buf := make([]byte, DropThreshold/2)
buf[0] = 1
_ = buf
}
}
func BenchmarkBytesPool_GetAll(b *testing.B) {
for i := range b.N {
bytesPool.Put(bytesPool.GetSized(sizes[i%len(sizes)]))
}
}
func BenchmarkBytesPool_GetAllUnsized(b *testing.B) {
for i := range b.N {
bytesPool.Put(slices.Grow(bytesPool.Get(), sizes[i%len(sizes)]))
}
}
func BenchmarkBytesPool_MakeAll(b *testing.B) {
for i := range b.N {
_ = make([]byte, sizes[i%len(sizes)])
}
}
func BenchmarkBytesPoolWithMemory(b *testing.B) {
pool := GetBytesPoolWithUniqueMemory()
for i := range b.N {
pool.Put(slices.Grow(pool.Get(), sizes[i%len(sizes)]))
}
}

View File

@@ -1,64 +0,0 @@
//go:build !production
package synk
import (
"os"
"os/signal"
"sync/atomic"
"time"
"github.com/rs/zerolog/log"
"github.com/yusing/godoxy/internal/utils/strutils"
)
type poolCounters struct {
num atomic.Uint64
size atomic.Uint64
}
var (
nonPooled poolCounters
dropped poolCounters
reused poolCounters
)
func addNonPooled(size int) {
nonPooled.num.Add(1)
nonPooled.size.Add(uint64(size))
}
func addReused(size int) {
reused.num.Add(1)
reused.size.Add(uint64(size))
}
func addDropped(size int) {
dropped.num.Add(1)
dropped.size.Add(uint64(size))
}
func initPoolStats() {
go func() {
statsTicker := time.NewTicker(5 * time.Second)
defer statsTicker.Stop()
sig := make(chan os.Signal, 1)
signal.Notify(sig, os.Interrupt)
for {
select {
case <-sig:
return
case <-statsTicker.C:
log.Info().
Uint64("numReused", reused.num.Load()).
Str("sizeReused", strutils.FormatByteSize(reused.size.Load())).
Uint64("numDropped", dropped.num.Load()).
Str("sizeDropped", strutils.FormatByteSize(dropped.size.Load())).
Uint64("numNonPooled", nonPooled.num.Load()).
Str("sizeNonPooled", strutils.FormatByteSize(nonPooled.size.Load())).
Msg("bytes pool stats")
}
}
}()
}

View File

@@ -1,8 +0,0 @@
//go:build production
package synk
func addNonPooled(size int) {}
func addDropped(size int) {}
func addReused(size int) {}
func initPoolStats() {}

View File

@@ -1,263 +0,0 @@
package synk
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestSized(t *testing.T) {
b := bytesPool.GetSized(2 * SizedPoolThreshold)
assert.Equal(t, cap(b), 2*SizedPoolThreshold)
bytesPool.Put(b)
assert.Equal(t, underlyingPtr(b), underlyingPtr(bytesPool.GetSized(SizedPoolThreshold)))
}
func TestUnsized(t *testing.T) {
b := bytesPool.Get()
assert.Equal(t, cap(b), UnsizedAvg)
bytesPool.Put(b)
assert.Equal(t, underlyingPtr(b), underlyingPtr(bytesPool.Get()))
}
func TestGetSizedExactMatch(t *testing.T) {
// Test exact size match reuse
size := SizedPoolThreshold
b1 := bytesPool.GetSized(size)
assert.Equal(t, size, len(b1))
assert.Equal(t, size, cap(b1))
// Put back into pool
bytesPool.Put(b1)
// Get same size - should reuse the same buffer
b2 := bytesPool.GetSized(size)
assert.Equal(t, size, len(b2))
assert.Equal(t, size, cap(b2))
assert.Equal(t, underlyingPtr(b1), underlyingPtr(b2))
}
func TestGetSizedBufferSplit(t *testing.T) {
// Test buffer splitting when capacity > requested size
largeSize := 2 * SizedPoolThreshold
requestedSize := SizedPoolThreshold
// Create a large buffer and put it in pool
b1 := bytesPool.GetSized(largeSize)
assert.Equal(t, largeSize, len(b1))
assert.Equal(t, largeSize, cap(b1))
bytesPool.Put(b1)
// Request smaller size - should split the buffer
b2 := bytesPool.GetSized(requestedSize)
assert.Equal(t, requestedSize, len(b2))
assert.Equal(t, requestedSize, cap(b2)) // capacity should remain the original
assert.Equal(t, underlyingPtr(b1), underlyingPtr(b2))
// The remaining part should be put back in pool
// Request the remaining size to verify
remainingSize := largeSize - requestedSize
b3 := bytesPool.GetSized(remainingSize)
assert.Equal(t, remainingSize, len(b3))
assert.Equal(t, remainingSize, cap(b3))
// Verify the remaining buffer points to the correct memory location
originalPtr := underlyingPtr(b1)
remainingPtr := underlyingPtr(b3)
// The remaining buffer should start at original + requestedSize
expectedOffset := uintptr(originalPtr) + uintptr(requestedSize)
actualOffset := uintptr(remainingPtr)
assert.Equal(t, expectedOffset, actualOffset, "Remaining buffer should point to correct offset")
}
func TestGetSizedSmallRemainder(t *testing.T) {
// Test when remaining size is smaller than SizedPoolThreshold
poolSize := SizedPoolThreshold + 100 // Just slightly larger than threshold
requestedSize := SizedPoolThreshold
// Create buffer and put in pool
b1 := bytesPool.GetSized(poolSize)
bytesPool.Put(b1)
// Request size that leaves small remainder
b2 := bytesPool.GetSized(requestedSize)
assert.Equal(t, requestedSize, len(b2))
assert.Equal(t, requestedSize, cap(b2))
// The small remainder (100 bytes) should NOT be put back in sized pool
// Try to get the remainder size - should create new buffer
b3 := bytesPool.GetSized(100)
assert.Equal(t, 100, len(b3))
assert.Equal(t, 100, cap(b3))
assert.NotEqual(t, underlyingPtr(b2), underlyingPtr(b3))
}
func TestGetSizedSmallBufferBypass(t *testing.T) {
// Test that small buffers (< SizedPoolThreshold) don't use sized pool
smallSize := SizedPoolThreshold - 1
b1 := bytesPool.GetSized(smallSize)
assert.Equal(t, smallSize, len(b1))
assert.Equal(t, smallSize, cap(b1))
b2 := bytesPool.GetSized(smallSize)
assert.Equal(t, smallSize, len(b2))
assert.Equal(t, smallSize, cap(b2))
// Should be different buffers (not pooled)
assert.NotEqual(t, underlyingPtr(b1), underlyingPtr(b2))
}
func TestGetSizedBufferTooSmall(t *testing.T) {
// Test when pool buffer is smaller than requested size
smallSize := SizedPoolThreshold
largeSize := 2 * SizedPoolThreshold
// Put small buffer in pool
b1 := bytesPool.GetSized(smallSize)
bytesPool.Put(b1)
// Request larger size - should create new buffer, not reuse small one
b2 := bytesPool.GetSized(largeSize)
assert.Equal(t, largeSize, len(b2))
assert.Equal(t, largeSize, cap(b2))
assert.NotEqual(t, underlyingPtr(b1), underlyingPtr(b2))
// The small buffer should still be in pool
b3 := bytesPool.GetSized(smallSize)
assert.Equal(t, underlyingPtr(b1), underlyingPtr(b3))
}
func TestGetSizedMultipleSplits(t *testing.T) {
// Test multiple sequential splits of the same buffer
hugeSize := 4 * SizedPoolThreshold
splitSize := SizedPoolThreshold
// Create huge buffer
b1 := bytesPool.GetSized(hugeSize)
originalPtr := underlyingPtr(b1)
bytesPool.Put(b1)
// Split it into smaller pieces
pieces := make([][]byte, 0, 4)
for i := range 4 {
piece := bytesPool.GetSized(splitSize)
pieces = append(pieces, piece)
// Each piece should point to the correct offset
expectedOffset := uintptr(originalPtr) + uintptr(i*splitSize)
actualOffset := uintptr(underlyingPtr(piece))
assert.Equal(t, expectedOffset, actualOffset, "Piece %d should point to correct offset", i)
assert.Equal(t, splitSize, len(piece))
assert.Equal(t, splitSize, cap(piece))
}
// All pieces should have the same underlying capacity
for i, piece := range pieces {
assert.Equal(t, splitSize, cap(piece), "Piece %d should have correct capacity", i)
}
}
func TestGetSizedMemorySafety(t *testing.T) {
// Test that split buffers don't interfere with each other
totalSize := 3 * SizedPoolThreshold
firstSize := SizedPoolThreshold
// Create buffer and split it
b1 := bytesPool.GetSized(totalSize)
// Fill with test data
for i := range len(b1) {
b1[i] = byte(i % 256)
}
bytesPool.Put(b1)
// Get first part
first := bytesPool.GetSized(firstSize)
assert.Equal(t, firstSize, len(first))
// Verify data integrity
for i := range len(first) {
assert.Equal(t, byte(i%256), first[i], "Data should be preserved after split")
}
// Get remaining part
remainingSize := totalSize - firstSize
remaining := bytesPool.GetSized(remainingSize)
assert.Equal(t, remainingSize, len(remaining))
// Verify remaining data
for i := range len(remaining) {
expected := byte((i + firstSize) % 256)
assert.Equal(t, expected, remaining[i], "Remaining data should be preserved")
}
}
func TestGetSizedCapacityLimiting(t *testing.T) {
// Test that returned buffers have limited capacity to prevent overwrites
largeSize := 2 * SizedPoolThreshold
requestedSize := SizedPoolThreshold
// Create large buffer and put in pool
b1 := bytesPool.GetSized(largeSize)
bytesPool.Put(b1)
// Get smaller buffer from the split
b2 := bytesPool.GetSized(requestedSize)
assert.Equal(t, requestedSize, len(b2))
assert.Equal(t, requestedSize, cap(b2), "Returned buffer should have limited capacity")
// Try to append data - should not be able to overwrite beyond capacity
original := make([]byte, len(b2))
copy(original, b2)
// This append should force a new allocation since capacity is limited
b2 = append(b2, 1, 2, 3, 4, 5)
assert.Greater(t, len(b2), requestedSize, "Buffer should have grown")
// Get the remaining buffer to verify it wasn't affected
remainingSize := largeSize - requestedSize
b3 := bytesPool.GetSized(remainingSize)
assert.Equal(t, remainingSize, len(b3))
assert.Equal(t, remainingSize, cap(b3), "Remaining buffer should have limited capacity")
}
func TestGetSizedAppendSafety(t *testing.T) {
// Test that appending to returned buffer doesn't affect remaining buffer
totalSize := 4 * SizedPoolThreshold
firstSize := SizedPoolThreshold
// Create buffer with specific pattern
b1 := bytesPool.GetSized(totalSize)
for i := range len(b1) {
b1[i] = byte(100 + i%100)
}
bytesPool.Put(b1)
// Get first part
first := bytesPool.GetSized(firstSize)
assert.Equal(t, firstSize, cap(first), "First part should have limited capacity")
// Store original first part content
originalFirst := make([]byte, len(first))
copy(originalFirst, first)
// Get remaining part to establish its state
remaining := bytesPool.GetSized(SizedPoolThreshold)
// Store original remaining content
originalRemaining := make([]byte, len(remaining))
copy(originalRemaining, remaining)
// Now try to append to first - this should not affect remaining buffers
// since capacity is limited
first = append(first, make([]byte, 1000)...)
// Verify remaining buffer content is unchanged
for i := range len(originalRemaining) {
assert.Equal(t, originalRemaining[i], remaining[i],
"Remaining buffer should be unaffected by append to first buffer")
}
}

View File

@@ -1,49 +0,0 @@
package trie
import (
"sync/atomic"
)
// AnyValue is a wrapper of atomic.Value
// It is used to store values in trie nodes
// And allowed to assign to empty struct value when node
// is not an end node anymore
type AnyValue struct {
v atomic.Value
}
type zeroValue struct{}
var zero zeroValue
func (av *AnyValue) Store(v any) {
if v == nil {
av.v.Store(zero)
return
}
defer panicInvalidAssignment()
av.v.Store(v)
}
func (av *AnyValue) Swap(v any) any {
defer panicInvalidAssignment()
return av.v.Swap(v)
}
func (av *AnyValue) Load() any {
switch v := av.v.Load().(type) {
case zeroValue:
return nil
default:
return v
}
}
func (av *AnyValue) IsNil() bool {
switch v := av.v.Load().(type) {
case zeroValue:
return true // assigned nil manually
default:
return v == nil // uninitialized
}
}

View File

@@ -1,13 +0,0 @@
//go:build debug
package trie
import "fmt"
func panicInvalidAssignment() {
// assigned anything after manually assigning nil
// will panic because of type mismatch (zeroValue and v.(type))
if r := recover(); r != nil {
panic(fmt.Errorf("attempt to assign non-nil value on edge node or assigning mismatched type: %v", r))
}
}

Some files were not shown because too many files have changed in this diff Show More