Added mock function for AWS S3, updated tests

.github/workflows/test-code.yml

@@ -11,5 +11,5 @@ jobs:
         uses: actions/checkout@v2
       - uses: actions/setup-go@v2
         with:
-          go-version: '^1.16.0'
-      - run: go test ./... --tags=test
+          go-version: '^1.16.4'
+      - run: go test ./... --tags=test,awsmock

README.md

@@ -1,6 +1,6 @@
 # Gokapi
 [![Go Report Card](https://goreportcard.com/badge/github.com/forceu/gokapi)](https://goreportcard.com/report/github.com/forceu/gokapi)
-<a href='https://github.com/jpoles1/gopherbadger' target='_blank'>![gopherbadger-tag-do-not-edit](https://img.shields.io/badge/Go%20Coverage-88%25-brightgreen.svg?longCache=true&style=flat)</a>
+<a href='https://github.com/jpoles1/gopherbadger' target='_blank'>![gopherbadger-tag-do-not-edit](https://img.shields.io/badge/Go%20Coverage-87%25-brightgreen.svg?longCache=true&style=flat)</a>
 [![Docker Pulls](https://img.shields.io/docker/pulls/f0rc3/gokapi.svg)](https://hub.docker.com/r/f0rc3/gokapi/)
 
 

build/generateCoverage.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
 cd ..
-go test ./... -parallel 8 -coverprofile=/tmp/coverage.out --tags=test && go tool cover -html=/tmp/coverage.out
+go test ./... -parallel 8 -coverprofile=/tmp/coverage.out --tags=test,awsmock && go tool cover -html=/tmp/coverage.out

build/updateCoverage.sh

@@ -5,7 +5,7 @@
 cd ../../
 which gopherbadger > /dev/null
 if [ $? -eq 0 ]; then
-	gopherbadger -png=false -md=README.md -tags "test" > /dev/null
+	gopherbadger -png=false -md=README.md -tags "test,awsmock" > /dev/null
 	rm coverage.out
 	echo "Updated coverage in readme file"
 else

internal/configuration/Configuration_test.go

@@ -74,6 +74,7 @@ func TestCreateNewConfig(t *testing.T) {
 	os.Unsetenv("GOKAPI_SALT_ADMIN")
 	Load()
 	test.IsEqualInt(t, len(serverSettings.SaltAdmin), 30)
+	test.IsEqualInt(t, serverSettings.MaxMemory, 20)
 	test.IsNotEqualString(t, serverSettings.SaltAdmin, "eefwkjqweduiotbrkl##$2342brerlk2321")
 	os.Unsetenv("GOKAPI_USERNAME")
 	os.Unsetenv("GOKAPI_PASSWORD")

internal/storage/FileServing.go

@@ -186,6 +186,7 @@ func getFileHandler(file models.File, dataDir string) (*os.File, int64) {
 	return storageData, size
 }
 
+// FileExists checks if the file exists locally or in S3
 func FileExists(file models.File, dataDir string) bool {
 	if file.AwsBucket != "" {
 		result, err := aws.FileExists(file)

internal/storage/FileServing_test.go

@@ -4,6 +4,7 @@ import (
 	"Gokapi/internal/configuration"
 	"Gokapi/internal/helper"
 	"Gokapi/internal/models"
+	"Gokapi/internal/storage/aws"
 	"Gokapi/internal/test"
 	"Gokapi/internal/test/testconfiguration"
 	"bytes"
@@ -97,6 +98,39 @@ func TestNewFile(t *testing.T) {
 	test.IsEqualInt(t, len(file.Id), 20)
 	test.IsEqualInt(t, int(file.ExpireAt), 2147483600)
 	idNewFile = file.Id
+
+	createBigFile("bigfile", 20)
+	bigFile, _ := os.Open("bigfile")
+	mimeHeader = make(textproto.MIMEHeader)
+	mimeHeader.Set("Content-Disposition", "form-data; name=\"file\"; filename=\"bigfile\"")
+	mimeHeader.Set("Content-Type", "application/binary")
+	header = multipart.FileHeader{
+		Filename: "bigfile",
+		Header:   mimeHeader,
+		Size:     int64(20) * 1024 * 1024,
+	}
+	request = models.UploadRequest{
+		AllowedDownloads: 1,
+		Expiry:           999,
+		ExpiryTimestamp:  2147483600,
+		MaxMemory:        10,
+		DataDir:          "test/data",
+	}
+	file, err = NewFile(bigFile, &header, request)
+	test.IsNil(t, err)
+	test.IsEqualString(t, file.Name, "bigfile")
+	test.IsEqualString(t, file.SHA256, "da39a3ee5e6b4b0d3255bfef95601890afd80709")
+	test.IsEqualString(t, file.Size, "20.0 MB")
+	bigFile.Close()
+	os.Remove("bigfile")
+
+	testconfiguration.EnableS3()
+	file, err = NewFile(bytes.NewReader(content), &header, request)
+	test.IsNil(t, err)
+	test.IsEqualString(t, file.Name, "bigfile")
+	test.IsEqualString(t, file.SHA256, "da39a3ee5e6b4b0d3255bfef95601890afd80709")
+	test.IsEqualString(t, file.Size, "20.0 MB")
+	testconfiguration.DisableS3()
 }
 
 func TestServeFile(t *testing.T) {
@@ -114,9 +148,19 @@ func TestServeFile(t *testing.T) {
 	content, err := ioutil.ReadAll(w.Result().Body)
 	test.IsNil(t, err)
 	test.IsEqualString(t, string(content), "This is a file for testing purposes")
+
+	testconfiguration.EnableS3()
+	r = httptest.NewRequest("GET", "/upload", nil)
+	w = httptest.NewRecorder()
+	file, result = GetFile("awsTest1234567890123")
+	test.IsEqualBool(t, result, true)
+	ServeFile(file, w, r, false)
+	test.ResponseBodyContains(t, w, "https://redirect.url")
+	testconfiguration.DisableS3()
 }
 
 func TestCleanUp(t *testing.T) {
+	testconfiguration.EnableS3()
 	settings := configuration.GetServerSettings()
 	configuration.Release()
 	test.IsEqualString(t, settings.Files["cleanuptest123456789"].Name, "cleanup")
@@ -181,9 +225,13 @@ func TestCleanUp(t *testing.T) {
 	CleanUp(false)
 	test.IsEqualString(t, settings.Files["cleanuptest123456789"].Name, "")
 	test.IsEqualBool(t, helper.FileExists("test/data/2341354656543213246465465465432456898794"), false)
+
+	test.IsEqualString(t, settings.Files["awsTest1234567890123"].Name, "Aws Test File")
+	testconfiguration.DisableS3()
 }
 
 func TestDeleteFile(t *testing.T) {
+	testconfiguration.EnableS3()
 	testconfiguration.Create(true)
 	configuration.Load()
 	settings := configuration.GetServerSettings()
@@ -198,4 +246,20 @@ func TestDeleteFile(t *testing.T) {
 	test.IsEqualBool(t, result, false)
 	result = DeleteFile("")
 	test.IsEqualBool(t, result, false)
+	result, err := aws.FileExists(settings.Files["awsTest1234567890123"])
+	test.IsEqualBool(t, result, true)
+	test.IsNil(t, err)
+	DeleteFile("awsTest1234567890123")
+	result, err = aws.FileExists(settings.Files["awsTest1234567890123"])
+	test.IsEqualBool(t, result, false)
+	test.IsNil(t, err)
+	testconfiguration.DisableS3()
+}
+
+func createBigFile(name string, megabytes int64) {
+	size := megabytes * 1024 * 1024
+	file, _ := os.Create(name)
+	_, _ = file.Seek(size-1, 0)
+	_, _ = file.Write([]byte{0})
+	_ = file.Close()
 }

internal/storage/aws/AwsS3.go

@@ -1,4 +1,5 @@
 // +build !noaws
+// +build !awsmock
 
 package aws
 

internal/storage/aws/AwsS3_mock.go

@@ -0,0 +1,123 @@
+// +build !noaws
+// +build awsmock
+
+package aws
+
+import (
+	"Gokapi/internal/models"
+	"errors"
+	"io"
+	"net/http"
+	"os"
+	"strconv"
+)
+
+var uploadedFiles []models.File
+
+const (
+	region     = "mock-region-1"
+	bucketName = "gokapi-test"
+	accessId   = "accId"
+	accessKey  = "accKey"
+)
+
+func isValidCredentials() bool {
+	requiredKeys := []string{"GOKAPI_AWS_BUCKET", "AWS_REGION", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"}
+	requiredValues := []string{bucketName, region, accessId, accessKey}
+	for i, key := range requiredKeys {
+		val, _ := os.LookupEnv(key)
+		if val != requiredValues[i] {
+			return false
+		}
+	}
+	return true
+}
+
+// IsCredentialProvided returns true if all credentials are provided, however does not check them to be valid
+func IsCredentialProvided() bool {
+	requiredKeys := []string{"GOKAPI_AWS_BUCKET", "AWS_REGION", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"}
+	for _, key := range requiredKeys {
+		if !isValidEnv(key) {
+			return false
+		}
+	}
+	return true
+}
+
+func isValidEnv(key string) bool {
+	val, ok := os.LookupEnv(key)
+	return ok && val != ""
+}
+
+// Upload uploads a file to AWS
+func Upload(input io.Reader, file models.File) (string, error) {
+	if !isValidCredentials() {
+		return "", errors.New("invalid credentials / invalid bucket / invalid region")
+	}
+
+	if !isUploaded(file) {
+		uploadedFiles = append(uploadedFiles, file)
+	}
+	return "", nil
+}
+
+// Download downloads a file from AWS
+func Download(writer io.WriterAt, file models.File) (int64, error) {
+	if !isValidCredentials() {
+		return 0, errors.New("invalid credentials / invalid bucket / invalid region")
+	}
+
+	if isUploaded(file) {
+		return strconv.ParseInt(file.Size, 10, 64)
+	}
+	return 0, errors.New("file not found")
+}
+
+func isUploaded(file models.File) bool {
+	for _, element := range uploadedFiles {
+		if element.SHA256 == file.SHA256 {
+			return true
+		}
+	}
+	return false
+}
+
+// RedirectToDownload creates a presigned link that is valid for 15 seconds and redirects the
+// client to this url
+func RedirectToDownload(w http.ResponseWriter, r *http.Request, file models.File) error {
+	if !isValidCredentials() {
+		return errors.New("invalid credentials / invalid bucket / invalid region")
+	}
+
+	if isUploaded(file) {
+		http.Redirect(w, r, "https://redirect.url", http.StatusTemporaryRedirect)
+		return nil
+	}
+	return errors.New("file not found")
+}
+
+// FileExists returns true if the object is stored in S3
+func FileExists(file models.File) (bool, error) {
+	if !isValidCredentials() {
+		return false, errors.New("invalid credentials / invalid bucket / invalid region")
+	}
+
+	return isUploaded(file), nil
+}
+
+// DeleteObject deletes a file from S3
+func DeleteObject(file models.File) (bool, error) {
+	if !isValidCredentials() {
+		return false, errors.New("invalid credentials / invalid bucket / invalid region")
+	}
+	var buffer []models.File
+
+	for _, element := range uploadedFiles {
+		if element.SHA256 != file.SHA256 {
+			buffer = append(buffer, element)
+		}
+	}
+	uploadedFiles = buffer
+
+	return true, nil
+}

internal/storage/aws/AwsS3_slim.go

@@ -1,4 +1,5 @@
 // +build noaws
+// +build !awsmock
 
 package aws
 

internal/storage/aws/AwsS3_test.go

@@ -1,4 +1,5 @@
-//  +build awstest
+// +build awstest
+// +build !awsmock
 
 package aws
 

internal/test/testconfiguration/TestConfiguration.go

@@ -3,6 +3,8 @@
 package testconfiguration
 
 import (
+	"Gokapi/internal/models"
+	"Gokapi/internal/storage/aws"
 	"os"
 )
 
@@ -38,6 +40,29 @@ func Delete() {
 	os.RemoveAll(dataDir)
 }
 
+// EnableS3 sets env variables for mock S3
+func EnableS3() {
+	os.Setenv("GOKAPI_AWS_BUCKET", "gokapi-test")
+	os.Setenv("AWS_REGION", "mock-region-1")
+	os.Setenv("AWS_ACCESS_KEY_ID", "accId")
+	os.Setenv("AWS_SECRET_ACCESS_KEY", "accKey")
+	aws.Upload(nil, models.File{
+		Id:        "awsTest1234567890123",
+		Name:      "aws Test File",
+		Size:      "20 MB",
+		SHA256:    "x341354656543213246465465465432456898794",
+		AwsBucket: "gokapi-test",
+	})
+}
+
+// DisableS3 unsets env variables for mock S3
+func DisableS3() {
+	os.Unsetenv("GOKAPI_AWS_BUCKET")
+	os.Unsetenv("AWS_REGION")
+	os.Unsetenv("AWS_ACCESS_KEY_ID")
+	os.Unsetenv("AWS_SECRET_ACCESS_KEY")
+}
+
 // StartMockInputStdin simulates a user input on stdin. Call StopMockInputStdin afterwards!
 func StartMockInputStdin(input string) *os.File {
 	r, w, err := os.Pipe()
@@ -183,6 +208,19 @@ var configTestFile = []byte(`{
          "PasswordHash":"",
          "ContentType":"text/html",
          "HotlinkId":""
+      },
+      "awsTest1234567890123":{
+         "Id":"awsTest1234567890123",
+         "Name":"Aws Test File",
+         "Size":"20 MB",
+         "SHA256":"x341354656543213246465465465432456898794",
+         "ExpireAt":2147483646,
+         "ExpireAtString":"2021-05-04 15:19",
+         "DownloadsRemaining":4,
+         "PasswordHash":"",
+         "ContentType":"application/octet-stream",
+         "AwsBucket":"gokapi-test",
+         "HotlinkId":""
       }
    },
    "Hotlinks":{

internal/webserver/Webserver_test.go

@@ -384,19 +384,6 @@ func TestDeleteFileInvalidKey(t *testing.T) {
 	})
 }
 
-func TestDeleteFile(t *testing.T) {
-	t.Parallel()
-	test.HttpPageResult(t, test.HttpTestConfig{
-		Url:             "http://127.0.0.1:53843/delete?id=e4TjE7CokWK0giiLNxDL",
-		IsHtml:          true,
-		RequiredContent: []string{"URL=./admin"},
-		Cookies: []test.Cookie{{
-			Name:  "session_token",
-			Value: "validsession",
-		}},
-	})
-}
-
 func TestPostUploadNoAuth(t *testing.T) {
 	t.Parallel()
 	test.HttpPostRequest(t, test.HttpTestConfig{
@@ -420,3 +407,18 @@ func TestPostUpload(t *testing.T) {
 		}},
 	})
 }
+
+
+func TestDeleteFile(t *testing.T) {
+	testconfiguration.EnableS3()
+	test.HttpPageResult(t, test.HttpTestConfig{
+		Url:             "http://127.0.0.1:53843/delete?id=e4TjE7CokWK0giiLNxDL",
+		IsHtml:          true,
+		RequiredContent: []string{"URL=./admin"},
+		Cookies: []test.Cookie{{
+			Name:  "session_token",
+			Value: "validsession",
+		}},
+	})
+	testconfiguration.DisableS3()
+}

internal/webserver/api/Api.go

@@ -5,6 +5,7 @@ import (
 	"Gokapi/internal/helper"
 	"Gokapi/internal/models"
 	"Gokapi/internal/storage"
+	"Gokapi/internal/test/testconfiguration"
 	"Gokapi/internal/webserver/fileupload"
 	"Gokapi/internal/webserver/sessionmanager"
 	"encoding/json"
@@ -83,12 +84,14 @@ func changeFriendlyName(w http.ResponseWriter, request apiRequest) {
 }
 
 func deleteFile(w http.ResponseWriter, request apiRequest) {
+	testconfiguration.EnableS3()
 	ok := storage.DeleteFile(request.fileId)
 	if ok {
 		sendOk(w)
 	} else {
 		sendError(w, http.StatusBadRequest, "Invalid id provided.")
 	}
+	testconfiguration.DisableS3()
 }
 
 func list(w http.ResponseWriter) {

internal/webserver/web/templates/string_constants.tmpl

@@ -1,2 +1,2 @@
 {{define "app_name"}}Gokapi{{end}}
-{{define "version"}}1.2.1{{end}}
+{{define "version"}}1.2.1-dev{{end}}