diff --git a/storage/pkg/command/drivers.go b/storage/pkg/command/drivers.go index 6cc6eb1257..0ab35aa89a 100644 --- a/storage/pkg/command/drivers.go +++ b/storage/pkg/command/drivers.go @@ -101,5 +101,15 @@ func drivers(cfg *config.Config) map[string]interface{} { "bucket": cfg.Reva.Storages.S3.Bucket, "prefix": cfg.Reva.Storages.S3.Root, }, + "s3ng": map[string]interface{}{ + "root": cfg.Reva.Storages.Common.Root, + "enable_home": cfg.Reva.Storages.Common.EnableHome, + "user_layout": cfg.Reva.Storages.Common.UserLayout, + "s3.region": cfg.Reva.Storages.S3NG.Region, + "s3.access_key": cfg.Reva.Storages.S3NG.AccessKey, + "s3.secret_key": cfg.Reva.Storages.S3NG.SecretKey, + "s3.endpoint": cfg.Reva.Storages.S3NG.Endpoint, + "s3.bucket": cfg.Reva.Storages.S3NG.Bucket, + }, } } diff --git a/storage/pkg/command/storagehome.go b/storage/pkg/command/storagehome.go index 69da0b3090..0b5ad9300d 100644 --- a/storage/pkg/command/storagehome.go +++ b/storage/pkg/command/storagehome.go @@ -50,6 +50,7 @@ func StorageHome(cfg *config.Config) *cli.Command { cfg.Reva.Storages.Local.EnableHome = true cfg.Reva.Storages.OwnCloud.EnableHome = true cfg.Reva.Storages.S3.EnableHome = true + cfg.Reva.Storages.S3NG.EnableHome = true } rcfg := storageHomeConfigFromStruct(c, cfg) diff --git a/storage/pkg/command/storageusers.go b/storage/pkg/command/storageusers.go index 6740b7c8e4..68a63637ce 100644 --- a/storage/pkg/command/storageusers.go +++ b/storage/pkg/command/storageusers.go @@ -50,6 +50,7 @@ func StorageUsers(cfg *config.Config) *cli.Command { cfg.Reva.Storages.Local.EnableHome = true cfg.Reva.Storages.OwnCloud.EnableHome = true cfg.Reva.Storages.S3.EnableHome = true + cfg.Reva.Storages.S3NG.EnableHome = true } rcfg := storageUsersConfigFromStruct(c, cfg) diff --git a/storage/pkg/config/config.go b/storage/pkg/config/config.go index 98a95a243b..0ca9b1c9d6 100644 --- a/storage/pkg/config/config.go +++ b/storage/pkg/config/config.go @@ -164,6 +164,7 @@ type StorageConfig struct { Local DriverCommon OwnCloud DriverOwnCloud S3 DriverS3 + S3NG DriverS3NG Common DriverCommon OCIS DriverOCIS // TODO checksums ... figure out what that is supposed to do @@ -268,6 +269,17 @@ type DriverS3 struct { Bucket string } +// DriverS3NG defines the available s3ng storage driver configuration. +type DriverS3NG struct { + DriverCommon + + Region string + AccessKey string + SecretKey string + Endpoint string + Bucket string +} + // OIDC defines the available OpenID Connect configuration. type OIDC struct { Issuer string diff --git a/storage/pkg/flagset/drivers3ng.go b/storage/pkg/flagset/drivers3ng.go new file mode 100644 index 0000000000..b755ac28ea --- /dev/null +++ b/storage/pkg/flagset/drivers3ng.go @@ -0,0 +1,69 @@ +package flagset + +import ( + "github.com/micro/cli/v2" + "github.com/owncloud/ocis/ocis-pkg/flags" + "github.com/owncloud/ocis/storage/pkg/config" +) + +// DriverS3NGWithConfig applies cfg to the root flagset +func DriverS3NGWithConfig(cfg *config.Config) []cli.Flag { + return []cli.Flag{ + &cli.StringFlag{ + Name: "storage-s3ng-root", + Value: flags.OverrideDefaultString(cfg.Reva.Storages.Common.Root, "/var/tmp/ocis/storage/users"), + Usage: "the path to the local storage root", + EnvVars: []string{"STORAGE_DRIVER_S3NG_ROOT"}, + Destination: &cfg.Reva.Storages.Common.Root, + }, + &cli.BoolFlag{ + Name: "storage-s3ng-enable-home", + Value: flags.OverrideDefaultBool(cfg.Reva.Storages.Common.EnableHome, false), + Usage: "enable the creation of home storages", + EnvVars: []string{"STORAGE_DRIVER_S3NG_ENABLE_HOME"}, + Destination: &cfg.Reva.Storages.Common.EnableHome, + }, + &cli.StringFlag{ + Name: "storage-s3ng-layout", + Value: flags.OverrideDefaultString(cfg.Reva.Storages.Common.UserLayout, "{{.Id.OpaqueId}}"), + Usage: `"layout of the users home dir path on disk, in addition to {{.Username}}, {{.Mail}}, {{.Id.OpaqueId}}, {{.Id.Idp}} also supports prefixing dirs: "{{substr 0 1 .Username}}/{{.Username}}" will turn "Einstein" into "Ei/Einstein" `, + EnvVars: []string{"STORAGE_DRIVER_S3NG_LAYOUT"}, + Destination: &cfg.Reva.Storages.Common.UserLayout, + }, + &cli.StringFlag{ + Name: "storage-s3ng-region", + Value: "default", + Usage: `"the s3 region" `, + EnvVars: []string{"STORAGE_DRIVER_S3NG_REGION"}, + Destination: &cfg.Reva.Storages.S3NG.Region, + }, + &cli.StringFlag{ + Name: "storage-s3ng-accesskey", + Value: "", + Usage: `"the s3 access key" `, + EnvVars: []string{"STORAGE_DRIVER_S3NG_ACCESS_KEY"}, + Destination: &cfg.Reva.Storages.S3NG.AccessKey, + }, + &cli.StringFlag{ + Name: "storage-s3ng-secretkey", + Value: "", + Usage: `"the secret s3 api key" `, + EnvVars: []string{"STORAGE_DRIVER_S3NG_SECRET_KEY"}, + Destination: &cfg.Reva.Storages.S3NG.SecretKey, + }, + &cli.StringFlag{ + Name: "storage-s3ng-endpoint", + Value: "", + Usage: `"s3 compatible API endpoint" `, + EnvVars: []string{"STORAGE_DRIVER_S3NG_ENDPOINT"}, + Destination: &cfg.Reva.Storages.S3NG.Endpoint, + }, + &cli.StringFlag{ + Name: "storage-s3ng-bucket", + Value: "", + Usage: `"bucket where the data will be stored in`, + EnvVars: []string{"STORAGE_DRIVER_S3NG_BUCKET"}, + Destination: &cfg.Reva.Storages.S3NG.Bucket, + }, + } +} diff --git a/storage/pkg/flagset/storagehome.go b/storage/pkg/flagset/storagehome.go index 090366a9c0..1536795865 100644 --- a/storage/pkg/flagset/storagehome.go +++ b/storage/pkg/flagset/storagehome.go @@ -156,6 +156,7 @@ func StorageHomeWithConfig(cfg *config.Config) []cli.Flag { flags = append(flags, DriverLocalWithConfig(cfg)...) flags = append(flags, DriverOwnCloudWithConfig(cfg)...) flags = append(flags, DriverOCISWithConfig(cfg)...) + flags = append(flags, DriverS3NGWithConfig(cfg)...) return flags } diff --git a/storage/pkg/flagset/storageusers.go b/storage/pkg/flagset/storageusers.go index 70ba25b7c7..eaf18cd184 100644 --- a/storage/pkg/flagset/storageusers.go +++ b/storage/pkg/flagset/storageusers.go @@ -146,6 +146,7 @@ func StorageUsersWithConfig(cfg *config.Config) []cli.Flag { flags = append(flags, DriverLocalWithConfig(cfg)...) flags = append(flags, DriverOwnCloudWithConfig(cfg)...) flags = append(flags, DriverOCISWithConfig(cfg)...) + flags = append(flags, DriverS3NGWithConfig(cfg)...) return flags } diff --git a/storage/templates/CONFIGURATION.tmpl b/storage/templates/CONFIGURATION.tmpl index 6f1434ad8b..ec6fce608a 100644 --- a/storage/templates/CONFIGURATION.tmpl +++ b/storage/templates/CONFIGURATION.tmpl @@ -111,3 +111,7 @@ Example: Set the home and users Storage Provider to `ocis` ### Ocis Driver {{ template "option" (list .Options "DriverOCISWithConfig") -}} + +### S3ng Driver + +{{ template "option" (list .Options "DriverS3NGWithConfig") -}} diff --git a/tests/acceptance/docker/Makefile b/tests/acceptance/docker/Makefile index 19f771ccaf..aa144a07bd 100644 --- a/tests/acceptance/docker/Makefile +++ b/tests/acceptance/docker/Makefile @@ -54,6 +54,10 @@ help: @echo -e "\tmake localApiTests-apiAccountsHashDifficulty-ocis\t\t${BLUE}run apiAccountsHashDifficulty test suite${RESET}" @echo -e "\tmake localApiTests-apiBugDemonstration-ocis\t\t${BLUE}run apiBugDemonstration test suite${RESET}" @echo + @echo -e "${GREEN}Run full oCIS test suites against oCIS with s3ng storage:${RESET}\n" + @echo -e "\tmake localApiTests-apiBasic-s3ng\t\t${BLUE}run apiBasic test suite${RESET}" + @echo -e "\tmake localApiTests-apiOcisSpecific-s3ng\t\t${BLUE}run apiOcisSPecific test suite${RESET}" + @echo @echo -e "${GREEN}Run full oCIS test suites against oCIS with ownCloud storage:${RESET}\n" @echo -e "\tmake localApiTests-apiAccountsHashDifficulty-owncloud\t\t${BLUE}run apiAccountsHashDifficulty test suite${RESET}" @echo -e "\tmake localApiTests-apiBugDemonstration-owncloud\t${BLUE}run apiBugDemonstration test suite${RESET}" @@ -61,6 +65,9 @@ help: @echo -e "${GREEN}Run full ownCloud test suites against oCIS with oCIS storage:${RESET}\n" @echo -e "\tmake Core-API-Tests-ocis-storage-${RED}X${RESET}\t\t${BLUE}run test suite number X, where ${RED}X = 1 .. 10${RESET}" @echo + @echo -e "${GREEN}Run full ownCloud test suites against oCIS with s3ng storage:${RESET}\n" + @echo -e "\tmake Core-API-Tests-s3ng-storage-${RED}X${RESET}\t\t${BLUE}run test suite number X, where ${RED}X = 1 .. 10${RESET}" + @echo @echo -e "${GREEN}Run full ownCloud test suites against oCIS with ownCloud storage:${RESET}\n" @echo -e "\tmake Core-API-Tests-owncloud-storage-${RED}X${RESET}\t\t${BLUE}run test suite number X, where ${RED}X = 1 .. 10${RESET}" @echo @@ -70,6 +77,12 @@ help: @echo -e "\twhere ${YELLOW}BEHAT_FEATURE='...'${RESET} contains a relative path to the feature definition." @echo -e "\texample: ${RED}tests/acceptance/features/apiBugDemonstration/apiAuthOcs-ocsDELETEAuth.feature${RESET}" @echo + @echo -e "${GREEN}Run an oCIS feature test against oCIS with s3ng storage:${RESET}\n" + @echo -e "\tmake test-ocis-feature-s3ng-storage ${YELLOW}BEHAT_FEATURE='...'${RESET}\t${BLUE}run single feature test${RESET}" + @echo + @echo -e "\twhere ${YELLOW}BEHAT_FEATURE='...'${RESET} contains a relative path to the feature definition." + @echo -e "\texample: ${RED}tests/acceptance/features/apiOcisSpecific/apiAuthOcs-ocsDELETEAuth.feature${RESET}" + @echo @echo -e "${GREEN}Run an oCIS feature test against oCIS with owncloud storage:${RESET}\n" @echo -e "\tmake test-ocis-feature-owncloud-storage ${YELLOW}BEHAT_FEATURE='...'${RESET}\t${BLUE}run single feature test${RESET}" @echo @@ -82,6 +95,12 @@ help: @echo -e "\twhere ${YELLOW}BEHAT_FEATURE='...'${RESET} contains a relative path to the feature definition." @echo -e "\texample: ${RED}tests/acceptance/features/apiAuth/cors.feature${RESET}" @echo + @echo -e "${GREEN}Run an ownCloud feature test against oCIS with s3ng storage:${RESET}\n" + @echo -e "\tmake test-oc10-feature-s3ng-storage ${YELLOW}BEHAT_FEATURE='...'${RESET}\t${BLUE}run single feature test${RESET}" + @echo + @echo -e "\twhere ${YELLOW}BEHAT_FEATURE='...'${RESET} contains a relative path to the feature definition." + @echo -e "\texample: ${RED}tests/acceptance/features/apiAuth/cors.feature${RESET}" + @echo @echo -e "${GREEN}Run an ownCloud feature test against oCIS with owncloud storage:${RESET}\n" @echo -e "\tmake test-oc10-feature-owncloud-storage ${YELLOW}BEHAT_FEATURE='...'${RESET}\t${BLUE}run single feature test${RESET}" @echo @@ -105,6 +124,14 @@ test-ocis-feature-ocis-storage: ## test a ocis feature with oCIS storage, useage BEHAT_FEATURE=$(BEHAT_FEATURE) \ $(MAKE) --no-print-directory testSuite +.PHONY: test-ocis-feature-s3ng-storage +test-ocis-feature-s3ng-storage: ## test a ocis feature with s3ng storage, useage: make ... BEHAT_FEATURE='tests/acceptance/features/apiOcisSpecific/apiAuthOcs-ocsDELETEAuth.feature:7' + @TEST_SOURCE=ocis \ + STORAGE=s3ng \ + BEHAT_FEATURE=$(BEHAT_FEATURE) \ + START_CEPH=1 \ + $(MAKE) --no-print-directory testSuite + .PHONY: test-ocis-feature-owncloud-storage test-ocis-feature-owncloud-storage: ## test a ocis feature with oc10 storage, useage: make ... BEHAT_FEATURE='tests/acceptance/features/apiBugDemonstration/apiAuthOcs-ocsDELETEAuth.feature:7' @TEST_SOURCE=ocis \ @@ -119,6 +146,14 @@ test-oc10-feature-ocis-storage: ## test a oC10 feature with oCIS storage, useage BEHAT_FEATURE=$(BEHAT_FEATURE) \ $(MAKE) --no-print-directory testSuite +.PHONY: test-oc10-feature-s3ng-storage +test-oc10-feature-s3ng-storage: ## test a oC10 feature with s3ng storage, useage: make ... BEHAT_FEATURE='tests/acceptance/features/apiAuth/cors.feature' + @TEST_SOURCE=oc10 \ + STORAGE=s3ng \ + BEHAT_FEATURE=$(BEHAT_FEATURE) \ + START_CEPH=1 \ + $(MAKE) --no-print-directory testSuite + .PHONY: test-oc10-feature-owncloud-storage test-oc10-feature-owncloud-storage: ## test a oC10 feature with oc10 storage, useage: make ... BEHAT_FEATURE='tests/acceptance/features/apiAuth/cors.feature' @TEST_SOURCE=oc10 \ @@ -154,6 +189,23 @@ localApiTests-apiAccountsHashDifficulty-ocis: ## run apiAccountsHashDifficulty t BEHAT_SUITE=apiAccountsHashDifficulty \ $(MAKE) --no-print-directory testSuite +.PHONY: localApiTests-apiOcisSpecific-s3ng +localApiTests-apiOcisSpecific-s3ng: ## run apiOcisSPecific test suite with s3ng storage + @TEST_SOURCE=ocis \ + STORAGE=s3ng \ + BEHAT_SUITE=apiOcisSpecific \ + START_CEPH=1 \ + $(MAKE) --no-print-directory testSuite + + +.PHONY: localApiTests-apiBasic-s3ng +localApiTests-apiBasic-s3ng: ## run apiBasic test suite with s3ng storage + @TEST_SOURCE=ocis \ + STORAGE=s3ng \ + BEHAT_SUITE=apiBasic \ + START_CEPH=1 \ + $(MAKE) --no-print-directory testSuite + targets = $(addprefix Core-API-Tests-owncloud-storage-,$(PARTS)) .PHONY: $(targets) $(targets): @@ -174,7 +226,12 @@ $(targets): .PHONY: testSuite testSuite: clean-docker-container - @COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME) \ + @if [ -n "${START_CEPH}" ]; then \ + COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME) \ + COMPOSE_FILE=src/ceph.yml \ + docker-compose run start_ceph; \ + fi; \ + COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME) \ COMPOSE_FILE=$(COMPOSE_FILE) \ STORAGE=$(STORAGE) \ TEST_SOURCE=$(TEST_SOURCE) \ @@ -183,7 +240,7 @@ testSuite: clean-docker-container BEHAT_FEATURE=$(BEHAT_FEATURE) \ DIVIDE_INTO_NUM_PARTS=$(DIVIDE_INTO_NUM_PARTS) \ RUN_PART=$(RUN_PART) \ - docker-compose up -d --build --remove-orphans --force-recreate + docker-compose up -d --build --force-recreate .PHONY: show-test-logs show-test-logs: ## show logs of test diff --git a/tests/acceptance/docker/src/ceph.yml b/tests/acceptance/docker/src/ceph.yml new file mode 100644 index 0000000000..f894290ea6 --- /dev/null +++ b/tests/acceptance/docker/src/ceph.yml @@ -0,0 +1,17 @@ +services: + start_ceph: + image: dadarek/wait-for-dependencies + depends_on: + - ceph + command: ceph:5000 + ceph: + image: ceph/daemon + command: demo + environment: + MON_IP: 127.0.0.1 + CEPH_PUBLIC_NETWORK: 0.0.0.0/0 + CEPH_DEMO_UID: test-user + CEPH_DEMO_ACCESS_KEY: test + CEPH_DEMO_SECRET_KEY: test + CEPH_DEMO_BUCKET: test + RGW_NAME: ceph \ No newline at end of file diff --git a/tests/acceptance/docker/src/ocis-base.yml b/tests/acceptance/docker/src/ocis-base.yml index 9d57e7208e..76d6fb48cc 100644 --- a/tests/acceptance/docker/src/ocis-base.yml +++ b/tests/acceptance/docker/src/ocis-base.yml @@ -22,6 +22,12 @@ services: IDP_ISS: https://ocis-server:9200 IDP_TLS: "true" ACCOUNTS_HASH_DIFFICULTY: 4 + # s3ng specific settings + STORAGE_DRIVER_S3NG_ENDPOINT: http://ceph:8080 + STORAGE_DRIVER_S3NG_REGION: default + STORAGE_DRIVER_S3NG_ACCESS_KEY: test + STORAGE_DRIVER_S3NG_SECRET_KEY: test + STORAGE_DRIVER_S3NG_BUCKET: test volumes: - ../../../config:/drone/src/ocis/tests/config - oCISownCloud10testsuite:/srv diff --git a/tests/acceptance/docker/src/run-tests.sh b/tests/acceptance/docker/src/run-tests.sh index 062307439f..96d0f38760 100644 --- a/tests/acceptance/docker/src/run-tests.sh +++ b/tests/acceptance/docker/src/run-tests.sh @@ -62,6 +62,12 @@ then export DELETE_USER_DATA_CMD='rm -rf /srv/app/tmp/ocis/storage/users/nodes/root/* /srv/app/tmp/ocis/storage/users/nodes/*-*-*-*' export OCIS_REVA_DATA_ROOT='' export OCIS_SKELETON_STRATEGY='upload' + elif [ "$STORAGE" = "s3ng" ] + then + export BEHAT_FILTER_TAGS='~@skipOnOcis-S3NG-Storage' + export DELETE_USER_DATA_CMD='rm -rf /srv/app/tmp/ocis/storage/users/nodes/root/* /srv/app/tmp/ocis/storage/users/nodes/*-*-*-*' + export OCIS_REVA_DATA_ROOT='' + export OCIS_SKELETON_STRATEGY='upload' else echo "non existing storage selected" exit 1