From 884c2d94b52bea81f5fd3fc767cec332d39f58dc Mon Sep 17 00:00:00 2001 From: Marc Ole Bulling Date: Mon, 10 May 2021 14:37:29 +0200 Subject: [PATCH] Added AWS S3 support #14 --- .gitignore | 2 + build/entrypoint.sh | 25 ++-- build/go.mod | 1 + build/go.sum | 48 +++++++ cmd/gokapi/Main.go | 4 +- go.mod | 1 + go.sum | 23 ++++ internal/configuration/Configuration.go | 5 +- internal/configuration/Configuration_test.go | 3 + internal/environment/Environment.go | 26 ++-- internal/models/FileList.go | 13 +- internal/models/FileList_test.go | 4 +- internal/models/FileUpload.go | 2 + internal/storage/FileServing.go | 129 ++++++++++++++---- internal/storage/FileServing_test.go | 6 +- internal/storage/aws/AwsS3.go | 122 +++++++++++++++++ internal/storage/aws/AwsS3_slim.go | 43 ++++++ internal/storage/aws/AwsS3_test.go | 90 ++++++++++++ internal/test/TestHelper_test.go | 14 ++ internal/webserver/Webserver.go | 6 +- internal/webserver/api/Api.go | 8 +- internal/webserver/api/Api_test.go | 30 ++-- internal/webserver/fileupload/FileUpload.go | 8 +- .../webserver/fileupload/FileUpload_test.go | 2 +- .../web/templates/string_constants.tmpl | 2 +- 25 files changed, 530 insertions(+), 87 deletions(-) create mode 100644 build/go.sum create mode 100644 internal/storage/aws/AwsS3.go create mode 100644 internal/storage/aws/AwsS3_slim.go create mode 100644 internal/storage/aws/AwsS3_test.go diff --git a/.gitignore b/.gitignore index 94a928b..a4bb4f5 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ config/ data/ .idea/ Gokapi +build/*.zip +gokapi diff --git a/build/entrypoint.sh b/build/entrypoint.sh index 902db4d..91c4e46 100755 --- a/build/entrypoint.sh +++ b/build/entrypoint.sh @@ -2,26 +2,27 @@ set -e - targets=${@-"darwin/amd64 linux/amd64 linux/386 linux/arm linux/arm64 windows/amd64 windows/386"} cd /usr/src/myapp go generate ./... for target in $targets; do - os="$(echo $target | cut -d '/' -f1)" - arch="$(echo $target | cut -d '/' -f2)" - output="build/gokapi-${os}_${arch}" - if [ $os = "windows" ]; then - output+='.exe' - fi + for tag in "full" "noaws"; do + os="$(echo $target | cut -d '/' -f1)" + arch="$(echo $target | cut -d '/' -f2)" + output="build/gokapi_${tag}-${os}_${arch}" + if [ $os = "windows" ]; then + output+='.exe' + fi - echo "----> Building project for: $target" - GOOS=$os GOARCH=$arch CGO_ENABLED=0 go build -ldflags="-s -w -X 'Gokapi/internal/environment.Builder=Github Release Builder' -X 'Gokapi/internal/environment.BuildTime=$(date)'" -o $output Gokapi/cmd/gokapi - zip -j $output.zip $output > /dev/null - rm $output + echo "----> Building Gokapi ($tag) for $target" + GOOS=$os GOARCH=$arch CGO_ENABLED=0 go build -tags $tag -ldflags="-s -w -X 'Gokapi/internal/environment.Builder=Github Release Builder' -X 'Gokapi/internal/environment.BuildTime=$(date)'" -o $output Gokapi/cmd/gokapi + zip -j $output.zip $output >/dev/null + rm $output + done done echo "----> Build is complete. List of files at $release_path:" cd build/ -ls -l gokapi-* +ls -l gokapi_* diff --git a/build/go.mod b/build/go.mod index d5853b0..c35db4f 100644 --- a/build/go.mod +++ b/build/go.mod @@ -3,6 +3,7 @@ module Gokapi go 1.16 require ( + github.com/aws/aws-sdk-go v1.38.36 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect ) diff --git a/build/go.sum b/build/go.sum new file mode 100644 index 0000000..95004e2 --- /dev/null +++ b/build/go.sum @@ -0,0 +1,48 @@ +github.com/aws/aws-sdk-go v1.38.36/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 h1:+/+DxvQaYifJ+grD4klzrS5y+KJXldn/2YTl5JG+vZ8= +github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201029080932-201ba4db2418/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20191110171634-ad39bd3f0407/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +mvdan.cc/editorconfig v0.1.1-0.20200121172147-e40951bde157 h1:VBYz8greWWP8BDpRX0v7SDv/8rNlZVmdHdO4ZSsHj/E= +mvdan.cc/editorconfig v0.1.1-0.20200121172147-e40951bde157/go.mod h1:Ge4atmRUYqueGppvJ7JNrtqpqokoJEFxYbP0Z+WeKS8= +mvdan.cc/sh/v3 v3.2.4 h1:+fZaWcXWRjYAvqzEKoDhDM3DkxdDUykU2iw0VMKFe9s= +mvdan.cc/sh/v3 v3.2.4/go.mod h1:fPQmabBpREM/XQ9YXSU5ZFZ/Sm+PmKP9/vkFHgYKJEI= diff --git a/cmd/gokapi/Main.go b/cmd/gokapi/Main.go index eb0002d..2a1e25d 100644 --- a/cmd/gokapi/Main.go +++ b/cmd/gokapi/Main.go @@ -18,9 +18,9 @@ import ( // Version is the current version in readable form. // The go generate call below needs to be modified as well -const Version = "1.2.0" +const Version = "1.2.1-dev" -//go:generate sh "../../build/setVersionTemplate.sh" "1.2.0" +//go:generate sh "../../build/setVersionTemplate.sh" "1.2.1-dev" // Main routine that is called on startup func main() { diff --git a/go.mod b/go.mod index d5853b0..c35db4f 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module Gokapi go 1.16 require ( + github.com/aws/aws-sdk-go v1.38.36 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect ) diff --git a/go.sum b/go.sum index 0de219b..c119ab6 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,35 @@ +github.com/aws/aws-sdk-go v1.38.36 h1:MiqzQY/IOFTX/jmGse7ThafD0eyOC4TrCLv2KY1v+bI= +github.com/aws/aws-sdk-go v1.38.36/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/internal/configuration/Configuration.go b/internal/configuration/Configuration.go index e9044b7..bd4633b 100644 --- a/internal/configuration/Configuration.go +++ b/internal/configuration/Configuration.go @@ -58,6 +58,8 @@ type Configuration struct { SaltFiles string `json:"SaltFiles"` LengthId int `json:"LengthId"` DataDir string `json:"DataDir"` + AwsBucket string `json:"AwsBucket"` + MaxMemory int `json:"MaxMemory"` } // Load loads the configuration or creates the folder structure and a default configuration @@ -75,6 +77,8 @@ func Load() { err = decoder.Decode(&serverSettings) helper.Check(err) updateConfig() + serverSettings.AwsBucket = Environment.AwsBucketName + serverSettings.MaxMemory = Environment.MaxMemory helper.CreateDir(serverSettings.DataDir) } @@ -127,7 +131,6 @@ func updateConfig() { if serverSettings.ConfigVersion < 6 { serverSettings.ApiKeys = make(map[string]models.ApiKey) } - if serverSettings.ConfigVersion < currentConfigVersion { fmt.Println("Successfully upgraded database") serverSettings.ConfigVersion = currentConfigVersion diff --git a/internal/configuration/Configuration_test.go b/internal/configuration/Configuration_test.go index cf51865..6d6e33e 100644 --- a/internal/configuration/Configuration_test.go +++ b/internal/configuration/Configuration_test.go @@ -11,6 +11,7 @@ import ( func TestMain(m *testing.M) { testconfiguration.Create(false) + os.Setenv("GOKAPI_AWS_BUCKET", "bucket") exitVal := m.Run() testconfiguration.Delete() os.Exit(exitVal) @@ -67,6 +68,7 @@ func TestCreateNewConfig(t *testing.T) { test.IsEqualString(t, serverSettings.RedirectUrl, "http://test2.com") test.IsEqualString(t, serverSettings.AdminPassword, "5bbf5684437a4c658d2e0890d784694afb63f715") test.IsEqualString(t, HashPassword("testtest2", false), "5bbf5684437a4c658d2e0890d784694afb63f715") + test.IsEqualString(t, serverSettings.AwsBucket, "bucket") test.IsEqualInt(t, serverSettings.LengthId, 15) os.Remove("test/config.json") os.Unsetenv("GOKAPI_SALT_ADMIN") @@ -93,6 +95,7 @@ func TestUpgradeDb(t *testing.T) { test.IsEqualBool(t, serverSettings.DownloadStatus == nil, false) test.IsEqualString(t, serverSettings.Files["MgXJLe4XLfpXcL12ec4i"].ContentType, "application/octet-stream") test.IsEqualInt(t, serverSettings.ConfigVersion, currentConfigVersion) + test.IsEqualString(t, serverSettings.AwsBucket, "bucket") testconfiguration.Create(false) Load() } diff --git a/internal/environment/Environment.go b/internal/environment/Environment.go index 539326c..4f73e0a 100644 --- a/internal/environment/Environment.go +++ b/internal/environment/Environment.go @@ -28,13 +28,16 @@ type Environment struct { SaltAdmin string SaltFiles string LengthId int + AwsBucketName string + MaxMemory int } var defaultValues = defaultsEnvironment{ - CONFIG_DIR: "config", - CONFIG_FILE: "config.json", - DATA_DIR: "data", - LENGTH_ID: 15, + CONFIG_DIR: "config", + CONFIG_FILE: "config.json", + DATA_DIR: "data", + LENGTH_ID: 15, + MAX_MEMORY_UPLOAD_MB: 20, } // New parses the env variables @@ -57,6 +60,8 @@ func New() Environment { SaltFiles: envString("SALT_FILES"), WebserverLocalhost: envBool("LOCALHOST"), LengthId: envInt("LENGTH_ID", 5), + AwsBucketName: envString("AWS_BUCKET"), + MaxMemory: envInt("MAX_MEMORY_UPLOAD_MB", 5), } } @@ -121,10 +126,11 @@ func (structPointer *defaultsEnvironment) getInt(name string) int { } type defaultsEnvironment struct { - CONFIG_DIR string - CONFIG_FILE string - DATA_DIR string - SALT_ADMIN string - SALT_FILES string - LENGTH_ID int + CONFIG_DIR string + CONFIG_FILE string + DATA_DIR string + SALT_ADMIN string + SALT_FILES string + LENGTH_ID int + MAX_MEMORY_UPLOAD_MB int } diff --git a/internal/models/FileList.go b/internal/models/FileList.go index 336a231..f3e7d23 100644 --- a/internal/models/FileList.go +++ b/internal/models/FileList.go @@ -17,12 +17,7 @@ type File struct { PasswordHash string `json:"PasswordHash"` HotlinkId string `json:"HotlinkId"` ContentType string `json:"ContentType"` -} - -// Hotlink is a struct containing hotlink ids -type Hotlink struct { - Id string `json:"Id"` - FileId string `json:"FileId"` + AwsBucket string `json:"AwsBucket"` } // ToJsonResult converts the file info to a json String used for returning a result for an upload @@ -41,6 +36,12 @@ func (f *File) ToJsonResult(serverUrl string) string { return string(bytes) } +// Hotlink is a struct containing hotlink ids +type Hotlink struct { + Id string `json:"Id"` + FileId string `json:"FileId"` +} + // Result is the struct used for the result after an upload // swagger:model UploadResult type Result struct { diff --git a/internal/models/FileList_test.go b/internal/models/FileList_test.go index dcf52be..ea12244 100644 --- a/internal/models/FileList_test.go +++ b/internal/models/FileList_test.go @@ -16,7 +16,7 @@ func TestToJsonResult(t *testing.T) { DownloadsRemaining: 1, PasswordHash: "pwhash", HotlinkId: "hotlinkid", - ContentType: "test/html", + ContentType: "text/html", } - test.IsEqualString(t, file.ToJsonResult("serverurl/"), `{"Result":"OK","FileInfo":{"Id":"testId","Name":"testName","Size":"10 B","SHA256":"sha256","ExpireAt":50,"ExpireAtString":"future","DownloadsRemaining":1,"PasswordHash":"pwhash","HotlinkId":"hotlinkid","ContentType":"test/html"},"Url":"serverurl/d?id=","HotlinkUrl":"serverurl/hotlink/"}`) + test.IsEqualString(t, file.ToJsonResult("serverurl/"), `{"Result":"OK","FileInfo":{"Id":"testId","Name":"testName","Size":"10 B","SHA256":"sha256","ExpireAt":50,"ExpireAtString":"future","DownloadsRemaining":1,"PasswordHash":"pwhash","HotlinkId":"hotlinkid","ContentType":"text/html","AwsBucket":""},"Url":"serverurl/d?id=","HotlinkUrl":"serverurl/hotlink/"}`) } diff --git a/internal/models/FileUpload.go b/internal/models/FileUpload.go index 2852e5c..e46f56b 100644 --- a/internal/models/FileUpload.go +++ b/internal/models/FileUpload.go @@ -7,4 +7,6 @@ type UploadRequest struct { ExpiryTimestamp int64 Password string ExternalUrl string + MaxMemory int + DataDir string } diff --git a/internal/storage/FileServing.go b/internal/storage/FileServing.go index c92ec1a..d94eb7f 100644 --- a/internal/storage/FileServing.go +++ b/internal/storage/FileServing.go @@ -9,6 +9,8 @@ import ( "Gokapi/internal/configuration/downloadstatus" "Gokapi/internal/helper" "Gokapi/internal/models" + "Gokapi/internal/storage/aws" + "bytes" "crypto/sha1" "encoding/hex" "fmt" @@ -27,17 +29,13 @@ import ( // already exists, it is deduplicated. This function gathers information about the file, creates an ID and saves // it into the global configuration. func NewFile(fileContent io.Reader, fileHeader *multipart.FileHeader, uploadRequest models.UploadRequest) (models.File, error) { - fileBytes, err := ioutil.ReadAll(fileContent) - if err != nil { - return models.File{}, err - } id := helper.GenerateRandomString(configuration.GetLengthId()) - hash := sha1.New() - hash.Write(fileBytes) + reader, hash, tempFile := generateHash(fileContent, fileHeader, uploadRequest) + defer deleteTempFile(tempFile) file := models.File{ Id: id, Name: fileHeader.Filename, - SHA256: hex.EncodeToString(hash.Sum(nil)), + SHA256: hex.EncodeToString(hash), Size: helper.ByteCountSI(fileHeader.Size), ExpireAt: uploadRequest.ExpiryTimestamp, ExpireAtString: time.Unix(uploadRequest.ExpiryTimestamp, 0).Format("2006-01-02 15:04"), @@ -47,20 +45,59 @@ func NewFile(fileContent io.Reader, fileHeader *multipart.FileHeader, uploadRequ } addHotlink(&file) settings := configuration.GetServerSettings() - defer func() { configuration.ReleaseAndSave() }() - settings.Files[id] = file filename := settings.DataDir + "/" + file.SHA256 - if !helper.FileExists(settings.DataDir + "/" + file.SHA256) { - destinationFile, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) + dataDir := settings.DataDir + file.AwsBucket = settings.AwsBucket + settings.Files[id] = file + configuration.ReleaseAndSave() + if !aws.IsCredentialProvided() { + if !helper.FileExists(dataDir + "/" + file.SHA256) { + destinationFile, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return models.File{}, err + } + defer destinationFile.Close() + _, err = io.Copy(destinationFile, reader) + if err != nil { + return models.File{}, err + } + } + } else { + _, err := aws.Upload(reader, file) if err != nil { return models.File{}, err } - defer destinationFile.Close() - destinationFile.Write(fileBytes) } return file, nil } +func deleteTempFile(file *os.File) { + if file == nil { + return + } + err := os.Remove(file.Name()) + helper.Check(err) +} + +func generateHash(fileContent io.Reader, fileHeader *multipart.FileHeader, uploadRequest models.UploadRequest) (io.Reader, []byte, *os.File) { + hash := sha1.New() + if fileHeader.Size <= int64(uploadRequest.MaxMemory)*1024*1024 { + content, err := ioutil.ReadAll(fileContent) + helper.Check(err) + hash.Write(content) + return bytes.NewReader(content), hash.Sum(nil), nil + } + tempFile, err := os.CreateTemp(uploadRequest.DataDir, "upload") + helper.Check(err) + _, err = io.Copy(tempFile, fileContent) + helper.Check(err) + _, err = io.Copy(hash, tempFile) + helper.Check(err) + _, err = tempFile.Seek(0, io.SeekStart) + helper.Check(err) + return tempFile, hash.Sum(nil), tempFile +} + var imageFileExtensions = []string{".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".svg"} // If file is an image, create link for hotlinking @@ -92,7 +129,7 @@ func GetFile(id string) (models.File, bool) { if file.ExpireAt < time.Now().Unix() || file.DownloadsRemaining < 1 { return emptyResult, false } - if !helper.FileExists(settings.DataDir + "/" + file.SHA256) { + if !FileExists(file, settings.DataDir) { return emptyResult, false } return file, true @@ -116,20 +153,46 @@ func ServeFile(file models.File, w http.ResponseWriter, r *http.Request, forceDo file.DownloadsRemaining = file.DownloadsRemaining - 1 settings := configuration.GetServerSettings() settings.Files[file.Id] = file - storageData, err := os.OpenFile(settings.DataDir+"/"+file.SHA256, os.O_RDONLY, 0644) + dataDir := settings.DataDir configuration.Release() + + // If file is not stored on AWS + if file.AwsBucket == "" { + storageData, size := getFileHandler(file, dataDir) + defer storageData.Close() + if forceDownload { + w.Header().Set("Content-Disposition", "attachment; filename=\""+file.Name+"\"") + } + w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) + w.Header().Set("Content-Type", file.ContentType) + statusId := downloadstatus.SetDownload(file) + http.ServeContent(w, r, file.Name, time.Now(), storageData) + downloadstatus.SetComplete(statusId) + } else { + // If file is stored on AWS + downloadstatus.SetDownload(file) + err := aws.RedirectToDownload(w, r, file) + helper.Check(err) + // We are not setting a download complete status, as there is no reliable way to confirm that the + // file has been completely downloaded. It expires automatically after 24 hours. + } +} + +func getFileHandler(file models.File, dataDir string) (*os.File, int64) { + storageData, err := os.OpenFile(dataDir+"/"+file.SHA256, os.O_RDONLY, 0644) helper.Check(err) - defer storageData.Close() size, err := helper.GetFileSize(storageData) helper.Check(err) - if forceDownload { - w.Header().Set("Content-Disposition", "attachment; filename=\""+file.Name+"\"") + return storageData, size +} + +func FileExists(file models.File, dataDir string) bool { + if file.AwsBucket != "" { + result, err := aws.FileExists(file) + helper.Check(err) + return result } - w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) - w.Header().Set("Content-Type", file.ContentType) - statusId := downloadstatus.SetDownload(file) - http.ServeContent(w, r, file.Name, time.Now(), storageData) - downloadstatus.SetComplete(statusId) + return helper.FileExists(dataDir + "/" + file.SHA256) } // CleanUp removes expired files from the config and from the filesystem if they are not referenced by other files anymore @@ -141,7 +204,7 @@ func CleanUp(periodic bool) { wasItemDeleted := false settings := configuration.GetServerSettings() for key, element := range settings.Files { - fileExists := helper.FileExists(settings.DataDir + "/" + element.SHA256) + fileExists := FileExists(element, settings.DataDir) if (element.ExpireAt < timeNow || element.DownloadsRemaining < 1 || !fileExists) && !downloadstatus.IsCurrentlyDownloading(element, settings) { deleteFile := true for _, secondLoopElement := range settings.Files { @@ -150,10 +213,7 @@ func CleanUp(periodic bool) { } } if deleteFile && fileExists { - err := os.Remove(settings.DataDir + "/" + element.SHA256) - if err != nil { - fmt.Println(err) - } + deleteSource(element, settings.DataDir) } if element.HotlinkId != "" { delete(settings.Hotlinks, element.HotlinkId) @@ -174,6 +234,19 @@ func CleanUp(periodic bool) { } } +func deleteSource(file models.File, dataDir string) { + var err error + if file.AwsBucket != "" { + _, err = aws.DeleteObject(file) + helper.Check(err) + } else { + err = os.Remove(dataDir + "/" + file.SHA256) + } + if err != nil { + fmt.Println(err) + } +} + // DeleteFile is called when an admin requests deletion of a file // Returns true if file was deleted or false if ID did not exist func DeleteFile(keyId string) bool { diff --git a/internal/storage/FileServing_test.go b/internal/storage/FileServing_test.go index a503911..18264e6 100644 --- a/internal/storage/FileServing_test.go +++ b/internal/storage/FileServing_test.go @@ -72,7 +72,7 @@ func TestNewFile(t *testing.T) { content := []byte("This is a file for testing purposes") mimeHeader := make(textproto.MIMEHeader) mimeHeader.Set("Content-Disposition", "form-data; name=\"file\"; filename=\"test.dat\"") - mimeHeader.Set("Content-Type", "text") + mimeHeader.Set("Content-Type", "text/plain") header := multipart.FileHeader{ Filename: "test.dat", Header: mimeHeader, @@ -82,6 +82,8 @@ func TestNewFile(t *testing.T) { AllowedDownloads: 1, Expiry: 999, ExpiryTimestamp: 2147483600, + MaxMemory: 10, + DataDir: "test/data", } file, err := NewFile(bytes.NewReader(content), &header, request) test.IsNil(t, err) @@ -108,7 +110,7 @@ func TestServeFile(t *testing.T) { test.IsEqualString(t, w.Result().Header.Get("Content-Disposition"), "attachment; filename=\"test.dat\"") test.IsEqualString(t, w.Result().Header.Get("Content-Length"), "35") - test.IsEqualString(t, w.Result().Header.Get("Content-Type"), "text") + test.IsEqualString(t, w.Result().Header.Get("Content-Type"), "text/plain") content, err := ioutil.ReadAll(w.Result().Body) test.IsNil(t, err) test.IsEqualString(t, string(content), "This is a file for testing purposes") diff --git a/internal/storage/aws/AwsS3.go b/internal/storage/aws/AwsS3.go new file mode 100644 index 0000000..0dbb823 --- /dev/null +++ b/internal/storage/aws/AwsS3.go @@ -0,0 +1,122 @@ +// +build !noaws + +package aws + +import ( + "Gokapi/internal/models" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "io" + "net/http" + "os" + "time" +) + +// IsCredentialProvided returns true if all credentials are provided, however does not check them to be valid +func IsCredentialProvided() bool { + requiredKeys := []string{"GOKAPI_AWS_BUCKET", "AWS_REGION", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"} + for _, key := range requiredKeys { + if !isValidEnv(key) { + return false + } + } + return true +} + +func isValidEnv(key string) bool { + val, ok := os.LookupEnv(key) + return ok && val != "" +} + +// Upload uploads a file to AWS +func Upload(input io.Reader, file models.File) (string, error) { + session := session.Must(session.NewSession()) + uploader := s3manager.NewUploader(session) + + result, err := uploader.Upload(&s3manager.UploadInput{ + Bucket: aws.String(file.AwsBucket), + Key: aws.String(file.SHA256), + Body: input, + }) + if err != nil { + return "", err + } + return result.Location, nil +} + +// Download downloads a file from AWS +func Download(writer io.WriterAt, file models.File) (int64, error) { + session := session.Must(session.NewSession()) + downloader := s3manager.NewDownloader(session) + + size, err := downloader.Download(writer, &s3.GetObjectInput{ + Bucket: aws.String(file.AwsBucket), + Key: aws.String(file.SHA256), + }) + if err != nil { + return 0, err + } + return size, nil +} + +// RedirectToDownload creates a presigned link that is valid for 15 seconds and redirects the +// client to this url +func RedirectToDownload(w http.ResponseWriter, r *http.Request, file models.File) error { + session := session.Must(session.NewSession()) + s3svc := s3.New(session) + + req, _ := s3svc.GetObjectRequest(&s3.GetObjectInput{ + Bucket: aws.String(file.AwsBucket), + Key: aws.String(file.SHA256), + ResponseContentDisposition: aws.String("filename=" + file.Name), + }) + + url, err := req.Presign(15 * time.Second) + if err != nil { + return err + } + + http.Redirect(w, r, url, http.StatusTemporaryRedirect) + return nil +} + +// FileExists returns true if the object is stored in S3 +func FileExists(file models.File) (bool, error) { + session := session.Must(session.NewSession()) + svc := s3.New(session) + + _, err := svc.HeadObject(&s3.HeadObjectInput{ + Bucket: aws.String(file.AwsBucket), + Key: aws.String(file.SHA256), + }) + + if err != nil { + aerr, ok := err.(awserr.Error) + if ok { + if aerr.Code() == "NotFound" { + return false, nil + } + } + return false, err + } + return true, nil +} + +// DeleteObject deletes a file from S3 +func DeleteObject(file models.File) (bool, error) { + session := session.Must(session.NewSession()) + svc := s3.New(session) + + _, err := svc.DeleteObject(&s3.DeleteObjectInput{ + Bucket: aws.String(file.AwsBucket), + Key: aws.String(file.SHA256), + }) + + if err != nil { + return false, err + } + return true, nil +} diff --git a/internal/storage/aws/AwsS3_slim.go b/internal/storage/aws/AwsS3_slim.go new file mode 100644 index 0000000..b97d99a --- /dev/null +++ b/internal/storage/aws/AwsS3_slim.go @@ -0,0 +1,43 @@ +// +build noaws + +package aws + +import ( + "Gokapi/internal/models" + "errors" + "io" + "net/http" +) + +const errorString = "AWS not supported in this build" + +// IsCredentialProvided returns true if all credentials are provided, however does not check them to be valid +func IsCredentialProvided() bool { + return false +} + +// Upload uploads a file to AWS +func Upload(input io.Reader, file models.File) (string, error) { + return "", errors.New(errorString) +} + +// Download downloads a file from AWS +func Download(writer io.WriterAt, file models.File) (int64, error) { + return 0, errors.New(errorString) +} + +// RedirectToDownload creates a presigned link that is valid for 15 seconds and redirects the +// client to this url +func RedirectToDownload(w http.ResponseWriter, r *http.Request, file models.File) error { + return errors.New(errorString) +} + +// FileExists returns true if the object is stored in S3 +func FileExists(file models.File) (bool, error) { + return true, errors.New(errorString) +} + +// DeleteObject deletes a file from S3 +func DeleteObject(file models.File) (bool, error) { + return false, errors.New(errorString) +} diff --git a/internal/storage/aws/AwsS3_test.go b/internal/storage/aws/AwsS3_test.go new file mode 100644 index 0000000..d9ba9c9 --- /dev/null +++ b/internal/storage/aws/AwsS3_test.go @@ -0,0 +1,90 @@ +// +build awstest + +package aws + +import ( + "Gokapi/internal/helper" + "Gokapi/internal/models" + "Gokapi/internal/test" + "os" + "testing" +) + +var testFile, invalidFile, invalidBucket, invalidAll models.File + +func TestMain(m *testing.M) { + testFile.AwsBucket = "gokapi-test" + testFile.SHA256 = "testfile" + invalidFile.AwsBucket = "gokapi-test" + invalidFile.SHA256 = "invalid" + invalidBucket.AwsBucket = "invalid" + invalidBucket.SHA256 = "testfile" + invalidAll.AwsBucket = "invalid" + invalidAll.SHA256 = "invalid" + exitVal := m.Run() + os.Exit(exitVal) +} + +func TestUploadToAws(t *testing.T) { + os.WriteFile("test", []byte("testfile-content"), 0777) + file, _ := os.Open("test") + location, err := Upload(file, testFile) + test.IsNil(t, err) + test.IsNotEmpty(t, location) + os.Remove("test") +} + +func TestDownloadFromAws(t *testing.T) { + test.IsEqualBool(t, helper.FileExists("test"), false) + file, _ := os.Create("test") + size, err := Download(file, testFile) + test.IsNil(t, err) + test.IsEqualBool(t, size == 16, true) + test.IsEqualBool(t, helper.FileExists("test"), true) + content, _ := os.ReadFile("test") + test.IsEqualString(t, string(content), "testfile-content") + os.Remove("test") +} + +func TestFileExists(t *testing.T) { + result, err := FileExists(invalidFile) + test.IsEqualBool(t, result, false) + test.IsNil(t, err) + result, err = FileExists(invalidBucket) + test.IsEqualBool(t, result, false) + test.IsNotNil(t, err) + result, err = FileExists(invalidAll) + test.IsEqualBool(t, result, false) + test.IsNotNil(t, err) + result, err = FileExists(testFile) + test.IsEqualBool(t, result, true) + test.IsNil(t, err) +} + +func TestDeleteObject(t *testing.T) { + result, err := FileExists(testFile) + test.IsEqualBool(t, result, true) + test.IsNil(t, err) + result, err = DeleteObject(testFile) + test.IsEqualBool(t, result, true) + test.IsNil(t, err) + result, err = FileExists(testFile) + test.IsEqualBool(t, result, false) + test.IsNil(t, err) + result, err = DeleteObject(invalidFile) + test.IsEqualBool(t, result, true) + test.IsNil(t, err) +} + +func TestIsCredentialProvided(t *testing.T) { + os.Unsetenv("AWS_REGION") + os.Unsetenv("AWS_ACCESS_KEY_ID") + os.Unsetenv("AWS_SECRET_ACCESS_KEY") + test.IsEqualBool(t, IsCredentialProvided(), false) + os.Setenv("AWS_REGION", "valid") + test.IsEqualBool(t, IsCredentialProvided(), false) + os.Setenv("AWS_ACCESS_KEY_ID", "valid") + test.IsEqualBool(t, IsCredentialProvided(), false) + os.Setenv("AWS_SECRET_ACCESS_KEY", "valid") + test.IsEqualBool(t, IsCredentialProvided(), true) +} diff --git a/internal/test/TestHelper_test.go b/internal/test/TestHelper_test.go index 83c3043..21065bf 100644 --- a/internal/test/TestHelper_test.go +++ b/internal/test/TestHelper_test.go @@ -5,6 +5,7 @@ import ( "io" "log" "net/http" + "net/http/httptest" "os" "testing" "time" @@ -156,6 +157,19 @@ func TestHttpPostRequest(t *testing.T) { os.Remove("testfile") } +func TestResponseBodyContains(t *testing.T) { + mockT := MockTest{reference: t} + mockT.WantNoFail() + w := httptest.NewRecorder() + _, _ = io.WriteString(w, "TestContentWrite") + ResponseBodyContains(mockT, w, "TestContentWrite") + mockT.WantFail() + w = httptest.NewRecorder() + _, _ = io.WriteString(w, "TestContentWrite") + ResponseBodyContains(mockT, w, "invalid") + mockT.Check() +} + func startTestServer() { http.HandleFunc("/test", func(writer http.ResponseWriter, request *http.Request) { io.WriteString(writer, "TestContent\n") diff --git a/internal/webserver/Webserver.go b/internal/webserver/Webserver.go index 491a725..3192b12 100644 --- a/internal/webserver/Webserver.go +++ b/internal/webserver/Webserver.go @@ -52,6 +52,7 @@ var ( webserverRedirectUrl string webserverAdminName string webserverAdminPassword string + webserverMaxMemory int ) // Start the webserver on the port set in the config @@ -102,6 +103,7 @@ func initLocalVariables() { webserverRedirectUrl = settings.RedirectUrl webserverAdminName = settings.AdminName webserverAdminPassword = settings.AdminPassword + webserverMaxMemory = settings.MaxMemory configuration.Release() } @@ -185,7 +187,7 @@ func deleteApiKey(w http.ResponseWriter, r *http.Request) { // Handling of /api/ func processApi(w http.ResponseWriter, r *http.Request) { - api.Process(w, r) + api.Process(w, r, webserverMaxMemory) } // Handling of /login @@ -395,7 +397,7 @@ func uploadFile(w http.ResponseWriter, r *http.Request) { if !isAuthenticated(w, r, true) { return } - err := fileupload.Process(w, r, true) + err := fileupload.Process(w, r, true, webserverMaxMemory) responseError(w, err) } diff --git a/internal/webserver/api/Api.go b/internal/webserver/api/Api.go index 694a1a6..58dbb6d 100644 --- a/internal/webserver/api/Api.go +++ b/internal/webserver/api/Api.go @@ -17,7 +17,7 @@ import ( //go:generate echo "Copied openapi.json" // Process parses the request and executes the API call or returns an error message to the sender -func Process(w http.ResponseWriter, r *http.Request) { +func Process(w http.ResponseWriter, r *http.Request, maxMemory int) { w.Header().Set("cache-control", "no-store") w.Header().Set("Content-Type", "application/json; charset=UTF-8") request := parseRequest(r) @@ -28,7 +28,7 @@ func Process(w http.ResponseWriter, r *http.Request) { case "/files/list": list(w) case "/files/add": - upload(w, request) + upload(w, request, maxMemory) case "/files/delete": deleteFile(w, request) case "/auth/friendlyname": @@ -106,8 +106,8 @@ func list(w http.ResponseWriter) { _, _ = w.Write(result) } -func upload(w http.ResponseWriter, request apiRequest) { - err := fileupload.Process(w, request.request, false) +func upload(w http.ResponseWriter, request apiRequest, maxMemory int) { + err := fileupload.Process(w, request.request, false, maxMemory) if err != nil { sendError(w, http.StatusBadRequest, err.Error()) return diff --git a/internal/webserver/api/Api_test.go b/internal/webserver/api/Api_test.go index 0c18b1f..5a38e9a 100644 --- a/internal/webserver/api/Api_test.go +++ b/internal/webserver/api/Api_test.go @@ -24,6 +24,8 @@ func TestMain(m *testing.M) { os.Exit(exitVal) } +const maxMemory = 20 + var newKeyId string func TestNewKey(t *testing.T) { @@ -57,22 +59,22 @@ func TestIsValidApiKey(t *testing.T) { func TestProcess(t *testing.T) { w, r := getRecorder("GET", "/api/auth/friendlyname", nil, nil, nil) - Process(w, r) + Process(w, r, maxMemory) test.ResponseBodyContains(t, w, "{\"Result\":\"error\",\"ErrorMessage\":\"Unauthorized\"}") w, r = getRecorder("GET", "/api/invalid", nil, nil, nil) - Process(w, r) + Process(w, r, maxMemory) test.ResponseBodyContains(t, w, "Unauthorized") w, r = getRecorder("GET", "/api/invalid", nil, []test.Header{{ Name: "apikey", Value: "validkey", }}, nil) - Process(w, r) + Process(w, r, maxMemory) test.ResponseBodyContains(t, w, "Invalid request") w, r = getRecorder("GET", "/api/invalid", []test.Cookie{{ Name: "session_token", Value: "validsession", }}, nil, nil) - Process(w, r) + Process(w, r, maxMemory) test.ResponseBodyContains(t, w, "Invalid request") } @@ -83,23 +85,23 @@ func TestChangeFriendlyName(t *testing.T) { Name: "apikey", Value: "validkey", }}, nil) - Process(w, r) + Process(w, r, maxMemory) test.ResponseBodyContains(t, w, "Invalid api key provided.") w, r = getRecorder("GET", "/api/auth/friendlyname", nil, []test.Header{{ Name: "apikey", Value: "validkey"}, { Name: "apiKeyToModify", Value: "validkey"}}, nil) - Process(w, r) + Process(w, r, maxMemory) test.IsEqualInt(t, w.Code, 200) test.IsEqualString(t, settings.ApiKeys["validkey"].FriendlyName, "Unnamed key") w, r = getRecorder("GET", "/api/auth/friendlyname", nil, []test.Header{{ Name: "apikey", Value: "validkey"}, { Name: "apiKeyToModify", Value: "validkey"}, { Name: "friendlyName", Value: "NewName"}}, nil) - Process(w, r) + Process(w, r, maxMemory) test.IsEqualInt(t, w.Code, 200) test.IsEqualString(t, settings.ApiKeys["validkey"].FriendlyName, "NewName") w = httptest.NewRecorder() - Process(w, r) + Process(w, r, maxMemory) test.IsEqualInt(t, w.Code, 200) } @@ -110,7 +112,7 @@ func TestDeleteFile(t *testing.T) { Name: "apikey", Value: "validkey", }}, nil) - Process(w, r) + Process(w, r, maxMemory) test.ResponseBodyContains(t, w, "Invalid id provided.") w, r = getRecorder("GET", "/api/files/delete", nil, []test.Header{{ Name: "apikey", @@ -120,7 +122,7 @@ func TestDeleteFile(t *testing.T) { Value: "invalid", }, }, nil) - Process(w, r) + Process(w, r, maxMemory) test.ResponseBodyContains(t, w, "Invalid id provided.") test.IsEqualString(t, settings.Files["jpLXGJKigM4hjtA6T6sN2"].Id, "jpLXGJKigM4hjtA6T6sN2") w, r = getRecorder("GET", "/api/files/delete", nil, []test.Header{{ @@ -131,7 +133,7 @@ func TestDeleteFile(t *testing.T) { Value: "jpLXGJKigM4hjtA6T6sN2", }, }, nil) - Process(w, r) + Process(w, r, maxMemory) test.IsEqualInt(t, w.Code, 200) test.IsEqualString(t, settings.Files["jpLXGJKigM4hjtA6T6sN2"].Id, "") } @@ -152,7 +154,7 @@ func TestUpload(t *testing.T) { }}, body) r.Header.Add("Content-Type", writer.FormDataContentType()) - Process(w, r) + Process(w, r, maxMemory) response, err := io.ReadAll(w.Result().Body) test.IsNil(t, err) result := models.Result{} @@ -165,7 +167,7 @@ func TestUpload(t *testing.T) { Name: "apikey", Value: "validkey", }}, body) - Process(w, r) + Process(w, r, maxMemory) test.ResponseBodyContains(t, w, "Content-Type isn't multipart/form-data") test.IsEqualInt(t, w.Code, 400) } @@ -175,7 +177,7 @@ func TestList(t *testing.T) { Name: "apikey", Value: "validkey", }}, nil) - Process(w, r) + Process(w, r, maxMemory) test.IsEqualInt(t, w.Code, 200) test.ResponseBodyContains(t, w, "picture.jpg") } diff --git a/internal/webserver/fileupload/FileUpload.go b/internal/webserver/fileupload/FileUpload.go index 0fa9220..d16289f 100644 --- a/internal/webserver/fileupload/FileUpload.go +++ b/internal/webserver/fileupload/FileUpload.go @@ -12,8 +12,8 @@ import ( ) // Process processes a file upload request -func Process(w http.ResponseWriter, r *http.Request, isWeb bool) error { - err := r.ParseMultipartForm(20 * 1024 * 1024) +func Process(w http.ResponseWriter, r *http.Request, isWeb bool, maxMemory int) error { + err := r.ParseMultipartForm(int64(maxMemory) * 1024 * 1024) if err != nil { return err } @@ -59,6 +59,8 @@ func parseConfig(values formOrHeader, setNewDefaults bool) models.UploadRequest settings.DefaultPassword = password } externalUrl := settings.ServerUrl + dataDir := settings.DataDir + maxMemory := settings.MaxMemory configuration.Release() return models.UploadRequest{ AllowedDownloads: allowedDownloadsInt, @@ -66,6 +68,8 @@ func parseConfig(values formOrHeader, setNewDefaults bool) models.UploadRequest ExpiryTimestamp: time.Now().Add(time.Duration(expiryDaysInt) * time.Hour * 24).Unix(), Password: password, ExternalUrl: externalUrl, + MaxMemory: maxMemory, + DataDir: dataDir, } } diff --git a/internal/webserver/fileupload/FileUpload_test.go b/internal/webserver/fileupload/FileUpload_test.go index 248a8b8..ac47cf9 100644 --- a/internal/webserver/fileupload/FileUpload_test.go +++ b/internal/webserver/fileupload/FileUpload_test.go @@ -51,7 +51,7 @@ func TestParseConfig(t *testing.T) { func TestProcess(t *testing.T) { w := httptest.NewRecorder() r := getRecorder() - err := Process(w, r, false) + err := Process(w, r, false, 20) test.IsNil(t, err) resp := w.Result() body, _ := io.ReadAll(resp.Body) diff --git a/internal/webserver/web/templates/string_constants.tmpl b/internal/webserver/web/templates/string_constants.tmpl index f81b18a..c0c882d 100644 --- a/internal/webserver/web/templates/string_constants.tmpl +++ b/internal/webserver/web/templates/string_constants.tmpl @@ -1,2 +1,2 @@ {{define "app_name"}}Gokapi{{end}} -{{define "version"}}1.2.0{{end}} +{{define "version"}}1.2.1{{end}}