From 3be4ec70969176850cff7083970f220bb247442e Mon Sep 17 00:00:00 2001 From: Marc Ole Bulling Date: Mon, 10 May 2021 16:44:44 +0200 Subject: [PATCH] Added mock function for AWS S3, updated tests --- .github/workflows/test-code.yml | 4 +- README.md | 2 +- build/generateCoverage.sh | 2 +- build/updateCoverage.sh | 2 +- internal/configuration/Configuration_test.go | 1 + internal/storage/FileServing.go | 1 + internal/storage/FileServing_test.go | 64 +++++++++ internal/storage/aws/AwsS3.go | 1 + internal/storage/aws/AwsS3_mock.go | 123 ++++++++++++++++++ internal/storage/aws/AwsS3_slim.go | 1 + internal/storage/aws/AwsS3_test.go | 3 +- .../testconfiguration/TestConfiguration.go | 38 ++++++ internal/webserver/Webserver_test.go | 28 ++-- internal/webserver/api/Api.go | 3 + .../web/templates/string_constants.tmpl | 2 +- 15 files changed, 255 insertions(+), 20 deletions(-) create mode 100644 internal/storage/aws/AwsS3_mock.go diff --git a/.github/workflows/test-code.yml b/.github/workflows/test-code.yml index ce85ace..1434460 100644 --- a/.github/workflows/test-code.yml +++ b/.github/workflows/test-code.yml @@ -11,5 +11,5 @@ jobs: uses: actions/checkout@v2 - uses: actions/setup-go@v2 with: - go-version: '^1.16.0' - - run: go test ./... --tags=test + go-version: '^1.16.4' + - run: go test ./... --tags=test,awsmock diff --git a/README.md b/README.md index 3884a33..2314555 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Gokapi [![Go Report Card](https://goreportcard.com/badge/github.com/forceu/gokapi)](https://goreportcard.com/report/github.com/forceu/gokapi) -![gopherbadger-tag-do-not-edit](https://img.shields.io/badge/Go%20Coverage-88%25-brightgreen.svg?longCache=true&style=flat) +![gopherbadger-tag-do-not-edit](https://img.shields.io/badge/Go%20Coverage-87%25-brightgreen.svg?longCache=true&style=flat) [![Docker Pulls](https://img.shields.io/docker/pulls/f0rc3/gokapi.svg)](https://hub.docker.com/r/f0rc3/gokapi/) diff --git a/build/generateCoverage.sh b/build/generateCoverage.sh index 7a099c4..f7dba07 100644 --- a/build/generateCoverage.sh +++ b/build/generateCoverage.sh @@ -1,3 +1,3 @@ #!/bin/sh cd .. -go test ./... -parallel 8 -coverprofile=/tmp/coverage.out --tags=test && go tool cover -html=/tmp/coverage.out +go test ./... -parallel 8 -coverprofile=/tmp/coverage.out --tags=test,awsmock && go tool cover -html=/tmp/coverage.out diff --git a/build/updateCoverage.sh b/build/updateCoverage.sh index 5c32ed8..47f6116 100755 --- a/build/updateCoverage.sh +++ b/build/updateCoverage.sh @@ -5,7 +5,7 @@ cd ../../ which gopherbadger > /dev/null if [ $? -eq 0 ]; then - gopherbadger -png=false -md=README.md -tags "test" > /dev/null + gopherbadger -png=false -md=README.md -tags "test,awsmock" > /dev/null rm coverage.out echo "Updated coverage in readme file" else diff --git a/internal/configuration/Configuration_test.go b/internal/configuration/Configuration_test.go index 6d6e33e..c1df9a5 100644 --- a/internal/configuration/Configuration_test.go +++ b/internal/configuration/Configuration_test.go @@ -74,6 +74,7 @@ func TestCreateNewConfig(t *testing.T) { os.Unsetenv("GOKAPI_SALT_ADMIN") Load() test.IsEqualInt(t, len(serverSettings.SaltAdmin), 30) + test.IsEqualInt(t, serverSettings.MaxMemory, 20) test.IsNotEqualString(t, serverSettings.SaltAdmin, "eefwkjqweduiotbrkl##$2342brerlk2321") os.Unsetenv("GOKAPI_USERNAME") os.Unsetenv("GOKAPI_PASSWORD") diff --git a/internal/storage/FileServing.go b/internal/storage/FileServing.go index d94eb7f..732dfdc 100644 --- a/internal/storage/FileServing.go +++ b/internal/storage/FileServing.go @@ -186,6 +186,7 @@ func getFileHandler(file models.File, dataDir string) (*os.File, int64) { return storageData, size } +// FileExists checks if the file exists locally or in S3 func FileExists(file models.File, dataDir string) bool { if file.AwsBucket != "" { result, err := aws.FileExists(file) diff --git a/internal/storage/FileServing_test.go b/internal/storage/FileServing_test.go index 18264e6..6ef631c 100644 --- a/internal/storage/FileServing_test.go +++ b/internal/storage/FileServing_test.go @@ -4,6 +4,7 @@ import ( "Gokapi/internal/configuration" "Gokapi/internal/helper" "Gokapi/internal/models" + "Gokapi/internal/storage/aws" "Gokapi/internal/test" "Gokapi/internal/test/testconfiguration" "bytes" @@ -97,6 +98,39 @@ func TestNewFile(t *testing.T) { test.IsEqualInt(t, len(file.Id), 20) test.IsEqualInt(t, int(file.ExpireAt), 2147483600) idNewFile = file.Id + + createBigFile("bigfile", 20) + bigFile, _ := os.Open("bigfile") + mimeHeader = make(textproto.MIMEHeader) + mimeHeader.Set("Content-Disposition", "form-data; name=\"file\"; filename=\"bigfile\"") + mimeHeader.Set("Content-Type", "application/binary") + header = multipart.FileHeader{ + Filename: "bigfile", + Header: mimeHeader, + Size: int64(20) * 1024 * 1024, + } + request = models.UploadRequest{ + AllowedDownloads: 1, + Expiry: 999, + ExpiryTimestamp: 2147483600, + MaxMemory: 10, + DataDir: "test/data", + } + file, err = NewFile(bigFile, &header, request) + test.IsNil(t, err) + test.IsEqualString(t, file.Name, "bigfile") + test.IsEqualString(t, file.SHA256, "da39a3ee5e6b4b0d3255bfef95601890afd80709") + test.IsEqualString(t, file.Size, "20.0 MB") + bigFile.Close() + os.Remove("bigfile") + + testconfiguration.EnableS3() + file, err = NewFile(bytes.NewReader(content), &header, request) + test.IsNil(t, err) + test.IsEqualString(t, file.Name, "bigfile") + test.IsEqualString(t, file.SHA256, "da39a3ee5e6b4b0d3255bfef95601890afd80709") + test.IsEqualString(t, file.Size, "20.0 MB") + testconfiguration.DisableS3() } func TestServeFile(t *testing.T) { @@ -114,9 +148,19 @@ func TestServeFile(t *testing.T) { content, err := ioutil.ReadAll(w.Result().Body) test.IsNil(t, err) test.IsEqualString(t, string(content), "This is a file for testing purposes") + + testconfiguration.EnableS3() + r = httptest.NewRequest("GET", "/upload", nil) + w = httptest.NewRecorder() + file, result = GetFile("awsTest1234567890123") + test.IsEqualBool(t, result, true) + ServeFile(file, w, r, false) + test.ResponseBodyContains(t, w, "https://redirect.url") + testconfiguration.DisableS3() } func TestCleanUp(t *testing.T) { + testconfiguration.EnableS3() settings := configuration.GetServerSettings() configuration.Release() test.IsEqualString(t, settings.Files["cleanuptest123456789"].Name, "cleanup") @@ -181,9 +225,13 @@ func TestCleanUp(t *testing.T) { CleanUp(false) test.IsEqualString(t, settings.Files["cleanuptest123456789"].Name, "") test.IsEqualBool(t, helper.FileExists("test/data/2341354656543213246465465465432456898794"), false) + + test.IsEqualString(t, settings.Files["awsTest1234567890123"].Name, "Aws Test File") + testconfiguration.DisableS3() } func TestDeleteFile(t *testing.T) { + testconfiguration.EnableS3() testconfiguration.Create(true) configuration.Load() settings := configuration.GetServerSettings() @@ -198,4 +246,20 @@ func TestDeleteFile(t *testing.T) { test.IsEqualBool(t, result, false) result = DeleteFile("") test.IsEqualBool(t, result, false) + result, err := aws.FileExists(settings.Files["awsTest1234567890123"]) + test.IsEqualBool(t, result, true) + test.IsNil(t, err) + DeleteFile("awsTest1234567890123") + result, err = aws.FileExists(settings.Files["awsTest1234567890123"]) + test.IsEqualBool(t, result, false) + test.IsNil(t, err) + testconfiguration.DisableS3() +} + +func createBigFile(name string, megabytes int64) { + size := megabytes * 1024 * 1024 + file, _ := os.Create(name) + _, _ = file.Seek(size-1, 0) + _, _ = file.Write([]byte{0}) + _ = file.Close() } diff --git a/internal/storage/aws/AwsS3.go b/internal/storage/aws/AwsS3.go index 0dbb823..ffd5e72 100644 --- a/internal/storage/aws/AwsS3.go +++ b/internal/storage/aws/AwsS3.go @@ -1,4 +1,5 @@ // +build !noaws +// +build !awsmock package aws diff --git a/internal/storage/aws/AwsS3_mock.go b/internal/storage/aws/AwsS3_mock.go new file mode 100644 index 0000000..9963d92 --- /dev/null +++ b/internal/storage/aws/AwsS3_mock.go @@ -0,0 +1,123 @@ +// +build !noaws +// +build awsmock + +package aws + +import ( + "Gokapi/internal/models" + "errors" + "io" + "net/http" + "os" + "strconv" +) + +var uploadedFiles []models.File + +const ( + region = "mock-region-1" + bucketName = "gokapi-test" + accessId = "accId" + accessKey = "accKey" +) + +func isValidCredentials() bool { + requiredKeys := []string{"GOKAPI_AWS_BUCKET", "AWS_REGION", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"} + requiredValues := []string{bucketName, region, accessId, accessKey} + for i, key := range requiredKeys { + val, _ := os.LookupEnv(key) + if val != requiredValues[i] { + return false + } + } + return true +} + +// IsCredentialProvided returns true if all credentials are provided, however does not check them to be valid +func IsCredentialProvided() bool { + requiredKeys := []string{"GOKAPI_AWS_BUCKET", "AWS_REGION", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"} + for _, key := range requiredKeys { + if !isValidEnv(key) { + return false + } + } + return true +} + +func isValidEnv(key string) bool { + val, ok := os.LookupEnv(key) + return ok && val != "" +} + +// Upload uploads a file to AWS +func Upload(input io.Reader, file models.File) (string, error) { + if !isValidCredentials() { + return "", errors.New("invalid credentials / invalid bucket / invalid region") + } + + if !isUploaded(file) { + uploadedFiles = append(uploadedFiles, file) + } + return "", nil +} + +// Download downloads a file from AWS +func Download(writer io.WriterAt, file models.File) (int64, error) { + if !isValidCredentials() { + return 0, errors.New("invalid credentials / invalid bucket / invalid region") + } + + if isUploaded(file) { + return strconv.ParseInt(file.Size, 10, 64) + } + return 0, errors.New("file not found") +} + +func isUploaded(file models.File) bool { + for _, element := range uploadedFiles { + if element.SHA256 == file.SHA256 { + return true + } + } + return false +} + +// RedirectToDownload creates a presigned link that is valid for 15 seconds and redirects the +// client to this url +func RedirectToDownload(w http.ResponseWriter, r *http.Request, file models.File) error { + if !isValidCredentials() { + return errors.New("invalid credentials / invalid bucket / invalid region") + } + + if isUploaded(file) { + http.Redirect(w, r, "https://redirect.url", http.StatusTemporaryRedirect) + return nil + } + return errors.New("file not found") +} + +// FileExists returns true if the object is stored in S3 +func FileExists(file models.File) (bool, error) { + if !isValidCredentials() { + return false, errors.New("invalid credentials / invalid bucket / invalid region") + } + + return isUploaded(file), nil +} + +// DeleteObject deletes a file from S3 +func DeleteObject(file models.File) (bool, error) { + if !isValidCredentials() { + return false, errors.New("invalid credentials / invalid bucket / invalid region") + } + var buffer []models.File + + for _, element := range uploadedFiles { + if element.SHA256 != file.SHA256 { + buffer = append(buffer, element) + } + } + uploadedFiles = buffer + + return true, nil +} diff --git a/internal/storage/aws/AwsS3_slim.go b/internal/storage/aws/AwsS3_slim.go index b97d99a..9d48be7 100644 --- a/internal/storage/aws/AwsS3_slim.go +++ b/internal/storage/aws/AwsS3_slim.go @@ -1,4 +1,5 @@ // +build noaws +// +build !awsmock package aws diff --git a/internal/storage/aws/AwsS3_test.go b/internal/storage/aws/AwsS3_test.go index d9ba9c9..aed0a35 100644 --- a/internal/storage/aws/AwsS3_test.go +++ b/internal/storage/aws/AwsS3_test.go @@ -1,4 +1,5 @@ -// +build awstest +// +build awstest +// +build !awsmock package aws diff --git a/internal/test/testconfiguration/TestConfiguration.go b/internal/test/testconfiguration/TestConfiguration.go index 8744691..d0a6cf3 100644 --- a/internal/test/testconfiguration/TestConfiguration.go +++ b/internal/test/testconfiguration/TestConfiguration.go @@ -3,6 +3,8 @@ package testconfiguration import ( + "Gokapi/internal/models" + "Gokapi/internal/storage/aws" "os" ) @@ -38,6 +40,29 @@ func Delete() { os.RemoveAll(dataDir) } +// EnableS3 sets env variables for mock S3 +func EnableS3() { + os.Setenv("GOKAPI_AWS_BUCKET", "gokapi-test") + os.Setenv("AWS_REGION", "mock-region-1") + os.Setenv("AWS_ACCESS_KEY_ID", "accId") + os.Setenv("AWS_SECRET_ACCESS_KEY", "accKey") + aws.Upload(nil, models.File{ + Id: "awsTest1234567890123", + Name: "aws Test File", + Size: "20 MB", + SHA256: "x341354656543213246465465465432456898794", + AwsBucket: "gokapi-test", + }) +} + +// DisableS3 unsets env variables for mock S3 +func DisableS3() { + os.Unsetenv("GOKAPI_AWS_BUCKET") + os.Unsetenv("AWS_REGION") + os.Unsetenv("AWS_ACCESS_KEY_ID") + os.Unsetenv("AWS_SECRET_ACCESS_KEY") +} + // StartMockInputStdin simulates a user input on stdin. Call StopMockInputStdin afterwards! func StartMockInputStdin(input string) *os.File { r, w, err := os.Pipe() @@ -183,6 +208,19 @@ var configTestFile = []byte(`{ "PasswordHash":"", "ContentType":"text/html", "HotlinkId":"" + }, + "awsTest1234567890123":{ + "Id":"awsTest1234567890123", + "Name":"Aws Test File", + "Size":"20 MB", + "SHA256":"x341354656543213246465465465432456898794", + "ExpireAt":2147483646, + "ExpireAtString":"2021-05-04 15:19", + "DownloadsRemaining":4, + "PasswordHash":"", + "ContentType":"application/octet-stream", + "AwsBucket":"gokapi-test", + "HotlinkId":"" } }, "Hotlinks":{ diff --git a/internal/webserver/Webserver_test.go b/internal/webserver/Webserver_test.go index 1c40881..a24ad37 100644 --- a/internal/webserver/Webserver_test.go +++ b/internal/webserver/Webserver_test.go @@ -384,19 +384,6 @@ func TestDeleteFileInvalidKey(t *testing.T) { }) } -func TestDeleteFile(t *testing.T) { - t.Parallel() - test.HttpPageResult(t, test.HttpTestConfig{ - Url: "http://127.0.0.1:53843/delete?id=e4TjE7CokWK0giiLNxDL", - IsHtml: true, - RequiredContent: []string{"URL=./admin"}, - Cookies: []test.Cookie{{ - Name: "session_token", - Value: "validsession", - }}, - }) -} - func TestPostUploadNoAuth(t *testing.T) { t.Parallel() test.HttpPostRequest(t, test.HttpTestConfig{ @@ -420,3 +407,18 @@ func TestPostUpload(t *testing.T) { }}, }) } + + +func TestDeleteFile(t *testing.T) { + testconfiguration.EnableS3() + test.HttpPageResult(t, test.HttpTestConfig{ + Url: "http://127.0.0.1:53843/delete?id=e4TjE7CokWK0giiLNxDL", + IsHtml: true, + RequiredContent: []string{"URL=./admin"}, + Cookies: []test.Cookie{{ + Name: "session_token", + Value: "validsession", + }}, + }) + testconfiguration.DisableS3() +} \ No newline at end of file diff --git a/internal/webserver/api/Api.go b/internal/webserver/api/Api.go index 58dbb6d..967c2b0 100644 --- a/internal/webserver/api/Api.go +++ b/internal/webserver/api/Api.go @@ -5,6 +5,7 @@ import ( "Gokapi/internal/helper" "Gokapi/internal/models" "Gokapi/internal/storage" + "Gokapi/internal/test/testconfiguration" "Gokapi/internal/webserver/fileupload" "Gokapi/internal/webserver/sessionmanager" "encoding/json" @@ -83,12 +84,14 @@ func changeFriendlyName(w http.ResponseWriter, request apiRequest) { } func deleteFile(w http.ResponseWriter, request apiRequest) { + testconfiguration.EnableS3() ok := storage.DeleteFile(request.fileId) if ok { sendOk(w) } else { sendError(w, http.StatusBadRequest, "Invalid id provided.") } + testconfiguration.DisableS3() } func list(w http.ResponseWriter) { diff --git a/internal/webserver/web/templates/string_constants.tmpl b/internal/webserver/web/templates/string_constants.tmpl index c0c882d..1f8b972 100644 --- a/internal/webserver/web/templates/string_constants.tmpl +++ b/internal/webserver/web/templates/string_constants.tmpl @@ -1,2 +1,2 @@ {{define "app_name"}}Gokapi{{end}} -{{define "version"}}1.2.1{{end}} +{{define "version"}}1.2.1-dev{{end}}