From 8a33993d766fe0bf4c520ab5b546dcee9d44e0ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Sat, 12 Dec 2020 20:24:51 +0000 Subject: [PATCH] Make all paths configurable and default to MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- accounts/pkg/flagset/flagset.go | 2 +- changelog/unreleased/make-all-paths-configurable.md | 5 +++++ docs/extensions/storage/storages.md | 2 +- docs/ocis/getting-started.md | 2 +- konnectd/pkg/flagset/flagset.go | 2 +- ocis-pkg/indexer/index/cs3/non_unique.go | 2 +- ocis/docker-compose-eos-test.yml | 2 +- settings/pkg/flagset/flagset.go | 2 +- settings/ui/tests/run-acceptance-test.sh | 2 +- storage/pkg/command/frontend.go | 2 +- storage/pkg/command/sharing.go | 5 +++++ storage/pkg/command/storagehome.go | 1 + storage/pkg/command/storagemetadata.go | 1 + storage/pkg/command/storageusers.go | 1 + storage/pkg/config/config.go | 10 ++++++---- storage/pkg/flagset/driverlocal.go | 2 +- storage/pkg/flagset/driverowncloud.go | 4 ++-- storage/pkg/flagset/frontend.go | 11 +++++++++-- storage/pkg/flagset/sharing.go | 9 ++++++++- storage/pkg/flagset/storagehome.go | 7 +++++++ storage/pkg/flagset/storagemetadata.go | 9 ++++++++- storage/pkg/flagset/storageusers.go | 7 +++++++ store/pkg/flagset/flagset.go | 2 +- store/pkg/service/v0/service.go | 2 +- 24 files changed, 72 insertions(+), 22 deletions(-) create mode 100644 changelog/unreleased/make-all-paths-configurable.md diff --git a/accounts/pkg/flagset/flagset.go b/accounts/pkg/flagset/flagset.go index 8ff368719..b58bcce54 100644 --- a/accounts/pkg/flagset/flagset.go +++ b/accounts/pkg/flagset/flagset.go @@ -143,7 +143,7 @@ func ServerWithConfig(cfg *config.Config) []cli.Flag { &cli.StringFlag{ Name: "storage-disk-path", Value: "", - Usage: "Path on the local disk, e.g. /var/tmp/ocis-accounts", + Usage: "Path on the local disk, e.g. /var/tmp/ocis/accounts", EnvVars: []string{"ACCOUNTS_STORAGE_DISK_PATH"}, Destination: &cfg.Repo.Disk.Path, }, diff --git a/changelog/unreleased/make-all-paths-configurable.md b/changelog/unreleased/make-all-paths-configurable.md new file mode 100644 index 000000000..ef450114d --- /dev/null +++ b/changelog/unreleased/make-all-paths-configurable.md @@ -0,0 +1,5 @@ +Change: Make all paths configurable and default to `/var/tmp/ocis//...` + +Aligned all services to use a subdir of `/var/tmp/ocis/` by default. Also made some missing temp paths configurable via env vars and config flags. + +https://github.com/owncloud/ocis/pulls/1080 diff --git a/docs/extensions/storage/storages.md b/docs/extensions/storage/storages.md index 2992dfda0..a05917a32 100644 --- a/docs/extensions/storage/storages.md +++ b/docs/extensions/storage/storages.md @@ -120,7 +120,7 @@ To provide the other storage aspects we plan to implement a FUSE overlay filesys This is the current default storage driver. While it implements the file tree (using redis, including id based lookup), ETag propagation, trash, versions and sharing (including expiry) using the data directory layout of ownCloud 10 it has [known limitations](https://github.com/owncloud/core/issues/28095) that cannot be fixed without changing the actual layout on disk. To setup it up properly in a distributed fashion, the storage-home and the storage-oc need to share the same underlying FS. Their "data" counterparts also need access to the same shared FS. -For a simple docker-compose setup, you can create a volume which will be used by the "storage-storage-home", "storage-storage-home-data", "storage-storage-oc" and "storage-storage-oc-data" containers. Using the `owncloud/ocis` docker image, the volume would need to be hooked in the `/var/tmp/ocis` folder insde the containers. +For a simple docker-compose setup, you can create a volume which will be used by the "storage-storage-home", "storage-storage-home-data", "storage-storage-oc" and "storage-storage-oc-data" containers. Using the `owncloud/ocis` docker image, the volume would need to be hooked in the `/var/tmp/ocis` folder inside the containers. - tree provided by a POSIX filesystem - file layout is mapped to the old ownCloud 10 layout diff --git a/docs/ocis/getting-started.md b/docs/ocis/getting-started.md index a61c1d8f1..2c68f85e8 100644 --- a/docs/ocis/getting-started.md +++ b/docs/ocis/getting-started.md @@ -29,7 +29,7 @@ chmod +x ocis ./ocis server ``` -The default primary storage location is `/var/tmp/`. You can change that value by configuration. +The default primary storage location is `/var/tmp/ocis`. You can change that value by configuration. ### Docker diff --git a/konnectd/pkg/flagset/flagset.go b/konnectd/pkg/flagset/flagset.go index 243d3e316..0dce35291 100644 --- a/konnectd/pkg/flagset/flagset.go +++ b/konnectd/pkg/flagset/flagset.go @@ -251,7 +251,7 @@ func ServerWithConfig(cfg *config.Config) []cli.Flag { Name: "identifier-client-path", Usage: "Path to the identifier web client base folder", EnvVars: []string{"KONNECTD_IDENTIFIER_CLIENT_PATH"}, - Value: "/var/tmp/konnectd", + Value: "/var/tmp/ocis/konnectd", Destination: &cfg.Konnectd.IdentifierClientPath, }, &cli.StringFlag{ diff --git a/ocis-pkg/indexer/index/cs3/non_unique.go b/ocis-pkg/indexer/index/cs3/non_unique.go index c93376891..fd0e5e630 100644 --- a/ocis-pkg/indexer/index/cs3/non_unique.go +++ b/ocis-pkg/indexer/index/cs3/non_unique.go @@ -45,7 +45,7 @@ type NonUnique struct { } // NewNonUniqueIndexWithOptions instantiates a new NonUniqueIndex instance. -// /var/tmp/ocis-accounts/index.cs3/Pets/Bro* +// /var/tmp/ocis/accounts/index.cs3/Pets/Bro* // ├── Brown/ // │ └── rebef-123 -> /var/tmp/testfiles-395764020/pets/rebef-123 // ├── Green/ diff --git a/ocis/docker-compose-eos-test.yml b/ocis/docker-compose-eos-test.yml index a8a341be0..f796d8a4c 100644 --- a/ocis/docker-compose-eos-test.yml +++ b/ocis/docker-compose-eos-test.yml @@ -44,7 +44,7 @@ services: # make accounts use ocis storage driver # TODO provision metadata storage in eos and switch to cs3 backend for accounts ACCOUNTS_LOG_LEVEL: debug - ACCOUNTS_STORAGE_DISK_PATH: /var/tmp/ocis-accounts + ACCOUNTS_STORAGE_DISK_PATH: /var/tmp/ocis/accounts # TODO make id the default in ocis-storage STORAGE_DRIVER_EOS_LAYOUT: "{{substr 0 1 .Id.OpaqueId}}/{{.Id.OpaqueId}}" STORAGE_FRONTEND_PUBLIC_URL: https://${OCIS_DOMAIN:-localhost}:9200 diff --git a/settings/pkg/flagset/flagset.go b/settings/pkg/flagset/flagset.go index ab3adfdc1..f4d04aba4 100644 --- a/settings/pkg/flagset/flagset.go +++ b/settings/pkg/flagset/flagset.go @@ -173,7 +173,7 @@ func ServerWithConfig(cfg *config.Config) []cli.Flag { }, &cli.StringFlag{ Name: "data-path", - Value: "/var/tmp/ocis-settings", + Value: "/var/tmp/ocis/settings", Usage: "Mount path for the storage", EnvVars: []string{"SETTINGS_DATA_PATH"}, Destination: &cfg.Service.DataPath, diff --git a/settings/ui/tests/run-acceptance-test.sh b/settings/ui/tests/run-acceptance-test.sh index 939af0770..53f732533 100755 --- a/settings/ui/tests/run-acceptance-test.sh +++ b/settings/ui/tests/run-acceptance-test.sh @@ -51,7 +51,7 @@ cp -r "$WEB_PATH/tests" "./$testFolder" export NODE_TLS_REJECT_UNAUTHORIZED='0' export SERVER_HOST=${SERVER_HOST:-https://localhost:9200} export BACKEND_HOST=${BACKEND_HOST:-https://localhost:9200} -export OCIS_SETTINGS_STORE=${OCIS_SETTINGS_STORE:-"/var/tmp/ocis-settings"} +export OCIS_SETTINGS_STORE=${OCIS_SETTINGS_STORE:-"/var/tmp/ocis/settings"} export RUN_ON_OCIS=true export TEST_TAGS=${TEST_TAGS:-"not @skip"} diff --git a/storage/pkg/command/frontend.go b/storage/pkg/command/frontend.go index 60455238f..7b122057b 100644 --- a/storage/pkg/command/frontend.go +++ b/storage/pkg/command/frontend.go @@ -130,7 +130,7 @@ func Frontend(cfg *config.Config) *cli.Command { }, "ocdav": map[string]interface{}{ "prefix": cfg.Reva.Frontend.OCDavPrefix, - "chunk_folder": "/var/tmp/ocis/chunks", + "chunk_folder": cfg.Reva.OCDav.ChunkFolder, "files_namespace": cfg.Reva.OCDav.DavFilesNamespace, "webdav_namespace": cfg.Reva.OCDav.WebdavNamespace, "timeout": 86400, diff --git a/storage/pkg/command/sharing.go b/storage/pkg/command/sharing.go index 4d48fb8e3..b8daaf107 100644 --- a/storage/pkg/command/sharing.go +++ b/storage/pkg/command/sharing.go @@ -97,6 +97,11 @@ func Sharing(cfg *config.Config) *cli.Command { }, "publicshareprovider": map[string]interface{}{ "driver": cfg.Reva.Sharing.PublicDriver, + "drivers": map[string]interface{}{ + "json": map[string]interface{}{ + "file": cfg.Reva.Sharing.PublicJSONFile, + }, + }, }, }, }, diff --git a/storage/pkg/command/storagehome.go b/storage/pkg/command/storagehome.go index 3e7cee192..2815c6d59 100644 --- a/storage/pkg/command/storagehome.go +++ b/storage/pkg/command/storagehome.go @@ -102,6 +102,7 @@ func StorageHome(cfg *config.Config) *cli.Command { "mount_id": cfg.Reva.StorageHome.MountID, "expose_data_server": cfg.Reva.StorageHome.ExposeDataServer, "data_server_url": cfg.Reva.StorageHome.DataServerURL, + "tmp_folder": cfg.Reva.StorageHome.TempFolder, }, }, }, diff --git a/storage/pkg/command/storagemetadata.go b/storage/pkg/command/storagemetadata.go index c21277e75..51c5c790f 100644 --- a/storage/pkg/command/storagemetadata.go +++ b/storage/pkg/command/storagemetadata.go @@ -111,6 +111,7 @@ func StorageMetadata(cfg *config.Config) *cli.Command { "driver": cfg.Reva.StorageMetadata.Driver, "drivers": drivers(cfg), "data_server_url": cfg.Reva.StorageMetadata.DataServerURL, + "tmp_folder": cfg.Reva.StorageMetadata.TempFolder, }, }, }, diff --git a/storage/pkg/command/storageusers.go b/storage/pkg/command/storageusers.go index 6e752adc0..17c9a93fb 100644 --- a/storage/pkg/command/storageusers.go +++ b/storage/pkg/command/storageusers.go @@ -102,6 +102,7 @@ func StorageUsers(cfg *config.Config) *cli.Command { "mount_id": cfg.Reva.StorageUsers.MountID, "expose_data_server": cfg.Reva.StorageUsers.ExposeDataServer, "data_server_url": cfg.Reva.StorageUsers.DataServerURL, + "tmp_folder": cfg.Reva.StorageUsers.TempFolder, }, }, }, diff --git a/storage/pkg/config/config.go b/storage/pkg/config/config.go index d60a166fa..0a38f0f8f 100644 --- a/storage/pkg/config/config.go +++ b/storage/pkg/config/config.go @@ -36,9 +36,10 @@ type StorageRegistry struct { // Sharing defines the available sharing configuration. type Sharing struct { Port - UserDriver string - UserJSONFile string - PublicDriver string + UserDriver string + UserJSONFile string + PublicDriver string + PublicJSONFile string } // Port defines the available port configuration. @@ -82,7 +83,7 @@ type FrontendPort struct { DatagatewayPrefix string OCDavPrefix string OCSPrefix string - OCSSharePrefix string + OCSSharePrefix string PublicURL string Middleware Middleware } @@ -282,6 +283,7 @@ type LDAPSchema struct { // OCDav defines the available ocdav configuration. type OCDav struct { + ChunkFolder string WebdavNamespace string DavFilesNamespace string } diff --git a/storage/pkg/flagset/driverlocal.go b/storage/pkg/flagset/driverlocal.go index 6acd21030..6184fa2b3 100644 --- a/storage/pkg/flagset/driverlocal.go +++ b/storage/pkg/flagset/driverlocal.go @@ -10,7 +10,7 @@ func DriverLocalWithConfig(cfg *config.Config) []cli.Flag { return []cli.Flag{ &cli.StringFlag{ Name: "storage-local-root", - Value: "/var/tmp/ocis/local", + Value: "/var/tmp/ocis/storage/local", Usage: "the path to the local storage root", EnvVars: []string{"STORAGE_DRIVER_LOCAL_ROOT"}, Destination: &cfg.Reva.Storages.Local.Root, diff --git a/storage/pkg/flagset/driverowncloud.go b/storage/pkg/flagset/driverowncloud.go index c74729194..633c0b4df 100644 --- a/storage/pkg/flagset/driverowncloud.go +++ b/storage/pkg/flagset/driverowncloud.go @@ -10,14 +10,14 @@ func DriverOwnCloudWithConfig(cfg *config.Config) []cli.Flag { return []cli.Flag{ &cli.StringFlag{ Name: "storage-owncloud-datadir", - Value: "/var/tmp/ocis/owncloud", + Value: "/var/tmp/ocis/storage/owncloud", Usage: "the path to the owncloud data directory", EnvVars: []string{"STORAGE_DRIVER_OWNCLOUD_DATADIR"}, Destination: &cfg.Reva.Storages.OwnCloud.Root, }, &cli.StringFlag{ Name: "storage-owncloud-uploadinfo-dir", - Value: "/var/tmp/ocis/uploadinfo", + Value: "/var/tmp/ocis/storage/uploadinfo", Usage: "the path to the tus upload info directory", EnvVars: []string{"STORAGE_DRIVER_OWNCLOUD_UPLOADINFO_DIR"}, Destination: &cfg.Reva.Storages.OwnCloud.UploadInfoDir, diff --git a/storage/pkg/flagset/frontend.go b/storage/pkg/flagset/frontend.go index cddbc5e5d..6cfaf820a 100644 --- a/storage/pkg/flagset/frontend.go +++ b/storage/pkg/flagset/frontend.go @@ -30,11 +30,18 @@ func FrontendWithConfig(cfg *config.Config) []cli.Flag { // OCDav + &cli.StringFlag{ + Name: "chunk-folder", + Value: "/var/tmp/ocis/tmp/chunks", + Usage: "temp directory for chunked uploads", + EnvVars: []string{"STORAGE_CHUNK_FOLDER"}, + Destination: &cfg.Reva.OCDav.WebdavNamespace, + }, &cli.StringFlag{ Name: "webdav-namespace", Value: "/home/", Usage: "Namespace prefix for the /webdav endpoint", - EnvVars: []string{"WEBDAV_NAMESPACE"}, + EnvVars: []string{"STORAGE_WEBDAV_NAMESPACE"}, Destination: &cfg.Reva.OCDav.WebdavNamespace, }, @@ -44,7 +51,7 @@ func FrontendWithConfig(cfg *config.Config) []cli.Flag { Name: "dav-files-namespace", Value: "/users/", Usage: "Namespace prefix for the webdav /dav/files endpoint", - EnvVars: []string{"DAV_FILES_NAMESPACE"}, + EnvVars: []string{"STORAGE_DAV_FILES_NAMESPACE"}, Destination: &cfg.Reva.OCDav.DavFilesNamespace, }, diff --git a/storage/pkg/flagset/sharing.go b/storage/pkg/flagset/sharing.go index 3b6129f47..fac4fc0cc 100644 --- a/storage/pkg/flagset/sharing.go +++ b/storage/pkg/flagset/sharing.go @@ -51,7 +51,7 @@ func SharingWithConfig(cfg *config.Config) []cli.Flag { }, &cli.StringFlag{ Name: "user-json-file", - Value: "/var/tmp/ocis/shares.json", + Value: "/var/tmp/ocis/storage/shares.json", Usage: "file used to persist shares for the UserShareProvider", EnvVars: []string{"STORAGE_SHARING_USER_JSON_FILE"}, Destination: &cfg.Reva.Sharing.UserJSONFile, @@ -63,6 +63,13 @@ func SharingWithConfig(cfg *config.Config) []cli.Flag { EnvVars: []string{"STORAGE_SHARING_PUBLIC_DRIVER"}, Destination: &cfg.Reva.Sharing.PublicDriver, }, + &cli.StringFlag{ + Name: "public-json-file", + Value: "/var/tmp/ocis/storage/publicshares.json", + Usage: "file used to persist shares for the PublicShareProvider", + EnvVars: []string{"STORAGE_SHARING_PUBLIC_JSON_FILE"}, + Destination: &cfg.Reva.Sharing.PublicJSONFile, + }, } flags = append(flags, TracingWithConfig(cfg)...) diff --git a/storage/pkg/flagset/storagehome.go b/storage/pkg/flagset/storagehome.go index 72d0d0c54..b6c0fe9fc 100644 --- a/storage/pkg/flagset/storagehome.go +++ b/storage/pkg/flagset/storagehome.go @@ -111,6 +111,13 @@ func StorageHomeWithConfig(cfg *config.Config) []cli.Flag { EnvVars: []string{"STORAGE_HOME_HTTP_PREFIX"}, Destination: &cfg.Reva.StorageHome.HTTPPrefix, }, + &cli.StringFlag{ + Name: "tmp-folder", + Value: "/var/tmp/ocis/tmp/home", + Usage: "path to tmp folder", + EnvVars: []string{"STORAGE_HOME_TMP_FOLDER"}, + Destination: &cfg.Reva.StorageHome.TempFolder, + }, &cli.BoolFlag{ Name: "enable-home", Value: true, diff --git a/storage/pkg/flagset/storagemetadata.go b/storage/pkg/flagset/storagemetadata.go index f0585c023..811983447 100644 --- a/storage/pkg/flagset/storagemetadata.go +++ b/storage/pkg/flagset/storagemetadata.go @@ -50,6 +50,13 @@ func StorageMetadata(cfg *config.Config) []cli.Flag { EnvVars: []string{"STORAGE_METADATA_HTTP_ADDR"}, Destination: &cfg.Reva.StorageMetadata.HTTPAddr, }, + &cli.StringFlag{ + Name: "tmp-folder", + Value: "/var/tmp/ocis/tmp/metadata", + Usage: "path to tmp folder", + EnvVars: []string{"STORAGE_METADATA_TMP_FOLDER"}, + Destination: &cfg.Reva.StorageMetadata.TempFolder, + }, &cli.StringFlag{ Name: "driver", Value: "ocis", @@ -91,7 +98,7 @@ func StorageMetadata(cfg *config.Config) []cli.Flag { flags = append(flags, &cli.StringFlag{ Name: "storage-root", - Value: "/var/tmp/ocis/metadata", + Value: "/var/tmp/ocis/storage/metadata", Usage: "the path to the metadata storage root", EnvVars: []string{"STORAGE_METADATA_ROOT"}, Destination: &cfg.Reva.Storages.Common.Root, diff --git a/storage/pkg/flagset/storageusers.go b/storage/pkg/flagset/storageusers.go index c5a53d198..4e6c3be75 100644 --- a/storage/pkg/flagset/storageusers.go +++ b/storage/pkg/flagset/storageusers.go @@ -108,6 +108,13 @@ func StorageUsersWithConfig(cfg *config.Config) []cli.Flag { EnvVars: []string{"STORAGE_USERS_HTTP_PREFIX"}, Destination: &cfg.Reva.StorageUsers.HTTPPrefix, }, + &cli.StringFlag{ + Name: "tmp-folder", + Value: "/var/tmp/ocis/tmp/users", + Usage: "path to tmp folder", + EnvVars: []string{"STORAGE_USERS_TMP_FOLDER"}, + Destination: &cfg.Reva.StorageUsers.TempFolder, + }, // some drivers need to look up users at the gateway diff --git a/store/pkg/flagset/flagset.go b/store/pkg/flagset/flagset.go index b731319da..cfb8c7bd4 100644 --- a/store/pkg/flagset/flagset.go +++ b/store/pkg/flagset/flagset.go @@ -131,7 +131,7 @@ func ServerWithConfig(cfg *config.Config) []cli.Flag { }, &cli.StringFlag{ Name: "data-path", - Value: "/var/tmp/ocis-store", + Value: "/var/tmp/ocis/store", Usage: "location of the store data path", EnvVars: []string{"STORE_DATA_PATH"}, Destination: &cfg.Datapath, diff --git a/store/pkg/service/v0/service.go b/store/pkg/service/v0/service.go index c773cc66c..f8e8576a2 100644 --- a/store/pkg/service/v0/service.go +++ b/store/pkg/service/v0/service.go @@ -242,7 +242,7 @@ func (s *Service) Tables(ctx context.Context, in *proto.TablesRequest, out *prot } // TODO sanitize key. As it may contain invalid characters, such as slashes. -// file: /var/tmp/ocis-store/databases/{database}/{table}/{record.key}. +// file: /var/tmp/ocis/store/databases/{database}/{table}/{record.key}. func getID(database string, table string, key string) string { // TODO sanitize input. return filepath.Join(database, table, key)