From 6cffccb3bef35c95b275c500b52dda8a297cbbbb Mon Sep 17 00:00:00 2001 From: Marc Ole Bulling Date: Wed, 16 Jun 2021 19:44:27 +0200 Subject: [PATCH] Added logging #18, set Dockerfile to Go 1.16.5, minor changes --- Dockerfile | 2 +- build/Dockerfile | 2 +- build/setVersionTemplate.sh | 2 +- cmd/gokapi/Main.go | 2 + internal/configuration/Configuration.go | 2 + internal/logging/Logging.go | 71 +++++++++++++++++++ internal/logging/Logging_test.go | 66 +++++++++++++++++ internal/storage/FileServing.go | 2 + internal/test/TestHelper.go | 4 +- internal/webserver/api/Api.go | 4 +- .../sessionmanager/SessionManager.go | 4 +- 11 files changed, 151 insertions(+), 10 deletions(-) create mode 100644 internal/logging/Logging.go create mode 100644 internal/logging/Logging_test.go diff --git a/Dockerfile b/Dockerfile index d582522..c108834 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.16.4 AS build_base +FROM golang:1.16.5 AS build_base ## Usage: ## docker build . -t gokapi diff --git a/build/Dockerfile b/build/Dockerfile index f021ccf..25f63b7 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.16.4 +FROM golang:1.16.5 ## To compile: ## cd Gokapi/build/ diff --git a/build/setVersionTemplate.sh b/build/setVersionTemplate.sh index 4fdbc12..568dc8f 100755 --- a/build/setVersionTemplate.sh +++ b/build/setVersionTemplate.sh @@ -2,4 +2,4 @@ #Called by go generate #Sets the version number in the template automatically sed -i 's/{{define "version"}}.*{{end}}/{{define "version"}}'$1'{{end}}/g' ../../internal/webserver/web/templates/string_constants.tmpl -echo "Set version in web template" +echo "Version in web template set" diff --git a/cmd/gokapi/Main.go b/cmd/gokapi/Main.go index 400e4d9..b825f6f 100644 --- a/cmd/gokapi/Main.go +++ b/cmd/gokapi/Main.go @@ -9,6 +9,7 @@ import ( "Gokapi/internal/configuration/cloudconfig" "Gokapi/internal/environment" "Gokapi/internal/helper" + "Gokapi/internal/logging" "Gokapi/internal/storage" "Gokapi/internal/storage/cloudstorage/aws" "Gokapi/internal/webserver" @@ -44,6 +45,7 @@ func main() { fmt.Println("Saving new files to local storage") } go storage.CleanUp(true) + logging.AddString("Gokapi started") webserver.Start() } diff --git a/internal/configuration/Configuration.go b/internal/configuration/Configuration.go index 0c1e26e..a8053af 100644 --- a/internal/configuration/Configuration.go +++ b/internal/configuration/Configuration.go @@ -7,6 +7,7 @@ Loading and saving of the persistent configuration import ( "Gokapi/internal/environment" "Gokapi/internal/helper" + log "Gokapi/internal/logging" "Gokapi/internal/models" "crypto/sha1" "encoding/hex" @@ -78,6 +79,7 @@ func Load() { updateConfig() serverSettings.MaxMemory = Environment.MaxMemory helper.CreateDir(serverSettings.DataDir) + log.Init(Environment.ConfigDir) } // Lock locks configuration to prevent race conditions (blocking) diff --git a/internal/logging/Logging.go b/internal/logging/Logging.go new file mode 100644 index 0000000..e6a6fbc --- /dev/null +++ b/internal/logging/Logging.go @@ -0,0 +1,71 @@ +package logging + +import ( + "Gokapi/internal/helper" + "Gokapi/internal/models" + "fmt" + "net" + "net/http" + "os" + "strings" + "sync" + "time" +) + +var logPath = "config/log.txt" +var mutex sync.Mutex + +// Init sets the path where to write the log file to +func Init(filePath string) { + logPath = filePath + "/log.txt" +} + +// AddString adds a line to the logfile including the current date. Non-Blocking +func AddString(text string) { + go writeToFile(text) +} + +// AddDownload adds a line to the logfile when a download was requested. Non-Blocking +func AddDownload(file *models.File, r *http.Request) { + AddString(fmt.Sprintf("Download: Filename %s, IP %s, ID %s, Useragent %s", file.Name, getIpAddress(r), file.Id, r.UserAgent())) +} + +func writeToFile(text string) { + mutex.Lock() + file, err := os.OpenFile(logPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + helper.Check(err) + defer file.Close() + defer mutex.Unlock() + _, err = file.WriteString(time.Now().UTC().Format(time.RFC1123) + " " + text + "\n") + helper.Check(err) +} + +func getIpAddress(r *http.Request) string { + // Get IP from X-FORWARDED-FOR header + ips := r.Header.Get("X-FORWARDED-FOR") + splitIps := strings.Split(ips, ",") + for _, ip := range splitIps { + netIP := net.ParseIP(ip) + if netIP != nil { + return ip + } + } + + // Get IP from the X-REAL-IP header + ip := r.Header.Get("X-REAL-IP") + netIP := net.ParseIP(ip) + if netIP != nil { + return ip + } + + // Get IP from RemoteAddr + ip, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + return "Unknown IP" + } + netIP = net.ParseIP(ip) + if netIP != nil { + return ip + } + return "Unknown IP" +} diff --git a/internal/logging/Logging_test.go b/internal/logging/Logging_test.go new file mode 100644 index 0000000..873df62 --- /dev/null +++ b/internal/logging/Logging_test.go @@ -0,0 +1,66 @@ +package logging + +import ( + "Gokapi/internal/models" + "Gokapi/internal/test" + "Gokapi/internal/test/testconfiguration" + "fmt" + "io/ioutil" + "net/http/httptest" + "os" + "strings" + "testing" + "time" +) + +func TestMain(m *testing.M) { + testconfiguration.Create(false) + exitVal := m.Run() + testconfiguration.Delete() + os.Exit(exitVal) +} + +func TestGetIpAddress(t *testing.T) { + r := httptest.NewRequest("GET", "/test", nil) + test.IsEqualString(t, getIpAddress(r), "192.0.2.1") + r = httptest.NewRequest("GET", "/test", nil) + r.RemoteAddr = "127.0.0.1:1234" + test.IsEqualString(t, getIpAddress(r), "127.0.0.1") + r.RemoteAddr = "invalid" + test.IsEqualString(t, getIpAddress(r), "Unknown IP") + r.Header.Add("X-REAL-IP", "1.1.1.1") + test.IsEqualString(t, getIpAddress(r), "1.1.1.1") + r.Header.Add("X-FORWARDED-FOR", "1.1.1.2") + test.IsEqualString(t, getIpAddress(r), "1.1.1.2") +} + +func TestInit(t *testing.T) { + Init("test") + test.IsEqualString(t, logPath, "test/log.txt") +} + +func TestAddString(t *testing.T) { + test.FileDoesNotExist(t, "test/log.txt") + AddString("Hello") + // Need sleep, as AddString() is non-blocking + time.Sleep(500 * time.Millisecond) + test.FileExists(t, "test/log.txt") + content, _ := ioutil.ReadFile("test/log.txt") + test.IsEqualBool(t, strings.Contains(string(content), "UTC Hello"), true) +} + +func TestAddDownload(t *testing.T) { + file := models.File{ + Id: "testId", + Name: "testName", + } + r := httptest.NewRequest("GET", "/test", nil) + r.Header.Set("User-Agent", "testAgent") + r.Header.Add("X-REAL-IP", "1.1.1.1") + AddDownload(&file, r) + // Need sleep, as AddDownload() is non-blocking + time.Sleep(500 * time.Millisecond) + content, _ := ioutil.ReadFile("test/log.txt") + fmt.Println(string(content)) + test.IsEqualBool(t, strings.Contains(string(content), "UTC Download: Filename testName, IP 1.1.1.1, ID testId, Useragent testAgent"), true) +} diff --git a/internal/storage/FileServing.go b/internal/storage/FileServing.go index d77a28a..126210e 100644 --- a/internal/storage/FileServing.go +++ b/internal/storage/FileServing.go @@ -8,6 +8,7 @@ import ( "Gokapi/internal/configuration" "Gokapi/internal/configuration/downloadstatus" "Gokapi/internal/helper" + "Gokapi/internal/logging" "Gokapi/internal/models" "Gokapi/internal/storage/cloudstorage/aws" "bytes" @@ -170,6 +171,7 @@ func ServeFile(file models.File, w http.ResponseWriter, r *http.Request, forceDo settings.Files[file.Id] = file dataDir := settings.DataDir configuration.Release() + logging.AddDownload(&file, r) // If file is not stored on AWS if file.AwsBucket == "" { diff --git a/internal/test/TestHelper.go b/internal/test/TestHelper.go index 9917873..255d8f8 100644 --- a/internal/test/TestHelper.go +++ b/internal/test/TestHelper.go @@ -90,7 +90,7 @@ func IsNil(t MockT, got error) { func FileExists(t MockT, name string) { t.Helper() if !fileExists(name) { - t.Errorf("Assertion failed, file does not exist: %s, want: nil.", name) + t.Errorf("Assertion failed, file does not exist: %s, want: Exists.", name) } } @@ -98,7 +98,7 @@ func FileExists(t MockT, name string) { func FileDoesNotExist(t MockT, name string) { t.Helper() if fileExists(name) { - t.Errorf("Assertion failed, file exist: %s, want: nil.", name) + t.Errorf("Assertion failed, file exist: %s, want: Does not exist", name) } } diff --git a/internal/webserver/api/Api.go b/internal/webserver/api/Api.go index 58dbb6d..00a1c7f 100644 --- a/internal/webserver/api/Api.go +++ b/internal/webserver/api/Api.go @@ -120,9 +120,7 @@ func isValidKey(key string, modifyTime bool) bool { return false } settings := configuration.GetServerSettings() - defer func() { - configuration.Release() - }() + defer configuration.Release() savedKey, ok := settings.ApiKeys[key] if ok && savedKey.Id != "" { if modifyTime { diff --git a/internal/webserver/sessionmanager/SessionManager.go b/internal/webserver/sessionmanager/SessionManager.go index 370718e..ca0dcff 100644 --- a/internal/webserver/sessionmanager/SessionManager.go +++ b/internal/webserver/sessionmanager/SessionManager.go @@ -24,7 +24,7 @@ func IsValidSession(w http.ResponseWriter, r *http.Request) bool { sessionString := cookie.Value if sessionString != "" { settings := configuration.GetServerSettings() - defer func() { configuration.ReleaseAndSave() }() + defer configuration.ReleaseAndSave() _, ok := (settings.Sessions)[sessionString] if ok { return useSession(w, sessionString, &settings.Sessions) @@ -56,7 +56,7 @@ func CreateSession(w http.ResponseWriter, sessions *map[string]models.Session) { if sessions == nil { settings := configuration.GetServerSettings() sessions = &settings.Sessions - defer func() { configuration.ReleaseAndSave() }() + defer configuration.ReleaseAndSave() } sessionString := helper.GenerateRandomString(60) (*sessions)[sessionString] = models.Session{