diff --git a/.drone.env b/.drone.env
index 109fb7a772..484f572405 100644
--- a/.drone.env
+++ b/.drone.env
@@ -1,7 +1,7 @@
# The test runner source for API tests
-CORE_COMMITID=4bbd91de68aefdf94c03d2920a3353702f856280
+CORE_COMMITID=38ee68e6358443e980ba5e7036cf1fb554443814
CORE_BRANCH=acceptance-test-changes-waiting-2021-11
# The test runner source for UI tests
-WEB_COMMITID=82e69d203bfb44a2a652a4c052fe7ce795bb1100
+WEB_COMMITID=ed4a8b32240b59666a6c2b162d95b50ff7191eb8
WEB_BRANCH=master
diff --git a/.drone.star b/.drone.star
index f718437f4d..7d03e5571a 100644
--- a/.drone.star
+++ b/.drone.star
@@ -1,4 +1,4 @@
-"""oCIS CI defintion
+"""oCIS CI definition
"""
# images
@@ -6,6 +6,7 @@ OC_CI_ALPINE = "owncloudci/alpine:latest"
OC_CI_GOLANG = "owncloudci/golang:1.17"
OC_CI_NODEJS = "owncloudci/nodejs:14"
OC_CI_PHP = "owncloudci/php:7.4"
+OC_CI_WAIT_FOR = "owncloudci/wait-for:latest"
MINIO_MC = "minio/mc:RELEASE.2021-10-07T04-19-58Z"
# configuration
@@ -577,7 +578,7 @@ def uiTestPipeline(ctx, filterTags, early_fail, runPart = 1, numberOfParts = 1,
"arch": "amd64",
},
"steps": skipIfUnchanged(ctx, "acceptance-tests") + restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin/ocis") +
- ocisServer(storage, accounts_hash_difficulty, [stepVolumeOC10Tests]) + [
+ ocisServer(storage, accounts_hash_difficulty, [stepVolumeOC10Tests]) + waitForSeleniumService() + waitForMiddlewareService() + [
{
"name": "webUITests",
"image": OC_CI_NODEJS,
@@ -594,6 +595,7 @@ def uiTestPipeline(ctx, filterTags, early_fail, runPart = 1, numberOfParts = 1,
"RUN_PART": runPart,
"DIVIDE_INTO_NUM_PARTS": numberOfParts,
"EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-webUI-on-%s-storage%s.md" % (storage.upper(), expectedFailuresFileFilterTags),
+ "MIDDLEWARE_HOST": "http://middleware:3000",
},
"commands": [
". /drone/src/.drone.env",
@@ -615,7 +617,7 @@ def uiTestPipeline(ctx, filterTags, early_fail, runPart = 1, numberOfParts = 1,
}],
},
] + failEarly(ctx, early_fail),
- "services": selenium(),
+ "services": selenium() + middlewareService(),
"volumes": [pipelineVolumeOC10Tests] +
[{
"name": "uploads",
@@ -643,7 +645,7 @@ def accountsUITests(ctx, storage = "ocis", accounts_hash_difficulty = 4):
"arch": "amd64",
},
"steps": skipIfUnchanged(ctx, "acceptance-tests") + restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin/ocis") +
- ocisServer(storage, accounts_hash_difficulty, [stepVolumeOC10Tests]) + [
+ ocisServer(storage, accounts_hash_difficulty, [stepVolumeOC10Tests]) + waitForSeleniumService() + waitForMiddlewareService() + [
{
"name": "WebUIAcceptanceTests",
"image": OC_CI_NODEJS,
@@ -658,6 +660,7 @@ def accountsUITests(ctx, storage = "ocis", accounts_hash_difficulty = 4):
"NODE_TLS_REJECT_UNAUTHORIZED": 0,
"WEB_PATH": "/srv/app/web",
"FEATURE_PATH": "/drone/src/accounts/ui/tests/acceptance/features",
+ "MIDDLEWARE_HOST": "http://middleware:3000",
},
"commands": [
". /drone/src/.drone.env",
@@ -678,7 +681,7 @@ def accountsUITests(ctx, storage = "ocis", accounts_hash_difficulty = 4):
}],
},
] + failEarly(ctx, early_fail),
- "services": selenium(),
+ "services": selenium() + middlewareService(),
"volumes": [stepVolumeOC10Tests] +
[{
"name": "uploads",
@@ -706,7 +709,7 @@ def settingsUITests(ctx, storage = "ocis", accounts_hash_difficulty = 4):
"arch": "amd64",
},
"steps": skipIfUnchanged(ctx, "acceptance-tests") + restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin/ocis") +
- ocisServer(storage, accounts_hash_difficulty, [stepVolumeOC10Tests]) + [
+ ocisServer(storage, accounts_hash_difficulty, [stepVolumeOC10Tests]) + waitForSeleniumService() + waitForMiddlewareService() + [
{
"name": "WebUIAcceptanceTests",
"image": OC_CI_NODEJS,
@@ -721,6 +724,7 @@ def settingsUITests(ctx, storage = "ocis", accounts_hash_difficulty = 4):
"NODE_TLS_REJECT_UNAUTHORIZED": 0,
"WEB_PATH": "/srv/app/web",
"FEATURE_PATH": "/drone/src/settings/ui/tests/acceptance/features",
+ "MIDDLEWARE_HOST": "http://middleware:3000",
},
"commands": [
". /drone/src/.drone.env",
@@ -746,7 +750,7 @@ def settingsUITests(ctx, storage = "ocis", accounts_hash_difficulty = 4):
"name": "redis",
"image": "redis:6-alpine",
},
- ] + selenium(),
+ ] + selenium() + middlewareService(),
"volumes": [stepVolumeOC10Tests] +
[{
"name": "uploads",
@@ -782,7 +786,7 @@ def failEarly(ctx, early_fail):
"image": "thegeeklab/drone-github-comment:1",
"settings": {
"message": ":boom: Acceptance test [${DRONE_STAGE_NAME}](${DRONE_BUILD_LINK}/${DRONE_STAGE_NUMBER}/1) failed. Further test are cancelled...",
- "key": "pr-${DRONE_PULL_REQUEST}", #TODO: we could delete the comment after a successfull CI run
+ "key": "pr-${DRONE_PULL_REQUEST}", #TODO: we could delete the comment after a successful CI run
"update": "true",
"api_key": {
"from_secret": "github_token",
@@ -1417,6 +1421,38 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = []):
},
]
+def middlewareService():
+ return [{
+ "name": "middleware",
+ "image": "owncloud/owncloud-test-middleware",
+ "pull": "always",
+ "environment": {
+ "BACKEND_HOST": "https://ocis-server:9200",
+ "OCIS_REVA_DATA_ROOT": "/srv/app/tmp/ocis/storage/owncloud/",
+ "RUN_ON_OCIS": "true",
+ "HOST": "middleware",
+ "REMOTE_UPLOAD_DIR": "/uploads",
+ "NODE_TLS_REJECT_UNAUTHORIZED": "0",
+ "MIDDLEWARE_HOST": "middleware",
+ },
+ "volumes": [{
+ "name": "uploads",
+ "path": "/uploads",
+ }, {
+ "name": "gopath",
+ "path": "/srv/app",
+ }],
+ }]
+
+def waitForMiddlewareService():
+ return [{
+ "name": "wait-for-middleware-service",
+ "image": OC_CI_WAIT_FOR,
+ "commands": [
+ "wait-for -it middleware:3000 -t 300",
+ ],
+ }]
+
def cloneCoreRepos():
return [
{
@@ -1459,6 +1495,15 @@ def selenium():
},
]
+def waitForSeleniumService():
+ return [{
+ "name": "wait-for-selenium-service",
+ "image": OC_CI_WAIT_FOR,
+ "commands": [
+ "wait-for -it selenium:4444 -t 300",
+ ],
+ }]
+
def build():
return [
{
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4094acfe47..10c72f6340 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,19 +2,71 @@
The following sections list the changes for unreleased.
-[unreleased]: https://github.com/owncloud/ocis/compare/v1.15.0...master
+[unreleased]: https://github.com/owncloud/ocis/compare/v1.16.0...master
+
+## Summary
+
+* Change - Update libre-graph-api to v0.3.0: [#2858](https://github.com/owncloud/ocis/pull/2858)
+* Change - Return not found when updating non existent space: [#2869](https://github.com/cs3org/reva/pull/2869)
+* Enhancement - Add new file url of the app provider to the ocs capabilities: [#2884](https://github.com/owncloud/ocis/pull/2884)
+* Enhancement - Support signature auth in the public share auth middleware: [#2831](https://github.com/owncloud/ocis/pull/2831)
+* Enhancement - Update REVA to xxx: [#2878](https://github.com/owncloud/ocis/pull/2878)
+
+## Details
+
+* Change - Update libre-graph-api to v0.3.0: [#2858](https://github.com/owncloud/ocis/pull/2858)
+
+ This updates the libre-graph-api to use the latest spec and types.
+
+ https://github.com/owncloud/ocis/pull/2858
+
+* Change - Return not found when updating non existent space: [#2869](https://github.com/cs3org/reva/pull/2869)
+
+ If a spaceid of a space which is updated doesn't exist, handle it as a not found error.
+
+ https://github.com/cs3org/reva/pull/2869
+
+* Enhancement - Add new file url of the app provider to the ocs capabilities: [#2884](https://github.com/owncloud/ocis/pull/2884)
+
+ We've added the new file capability of the app provider to the ocs capabilities, so that clients
+ can discover this url analogous to the app list and file open urls.
+
+ https://github.com/owncloud/ocis/pull/2884
+ https://github.com/cs3org/reva/pull/2379
+ https://github.com/owncloud/web/pull/5890#issuecomment-993905242
+
+* Enhancement - Support signature auth in the public share auth middleware: [#2831](https://github.com/owncloud/ocis/pull/2831)
+
+ Enabled public share requests to be authenticated using the public share signature.
+
+ https://github.com/owncloud/ocis/pull/2831
+
+* Enhancement - Update REVA to xxx: [#2878](https://github.com/owncloud/ocis/pull/2878)
+
+ Updated REVA to xxx This update includes:
+
+ * TODO: update changelog before oCIS release
+
+ https://github.com/owncloud/ocis/pull/2878
+# Changelog for [1.16.0] (2021-12-10)
+
+The following sections list the changes for 1.16.0.
+
+[1.16.0]: https://github.com/owncloud/ocis/compare/v1.15.0...v1.16.0
## Summary
* Bugfix - Fix claim selector based routing for basic auth: [#2779](https://github.com/owncloud/ocis/pull/2779)
* Bugfix - Disallow creation of a group with empty name via the OCS api: [#2825](https://github.com/owncloud/ocis/pull/2825)
* Bugfix - Fix using s3ng as the metadata storage backend: [#2807](https://github.com/owncloud/ocis/pull/2807)
+* Bugfix - Use the CS3api up- and download workflow for the accounts service: [#2837](https://github.com/owncloud/ocis/pull/2837)
* Change - Rename `APP_PROVIDER_BASIC_*` environment variables: [#2812](https://github.com/owncloud/ocis/pull/2812)
* Change - Restructure Configuration Parsing: [#2708](https://github.com/owncloud/ocis/pull/2708)
* Change - OIDC: fallback if IDP doesn't provide "preferred_username" claim: [#2644](https://github.com/owncloud/ocis/issues/2644)
* Enhancement - Cleanup ocis-pkg config: [#2813](https://github.com/owncloud/ocis/pull/2813)
* Enhancement - Correct shutdown of services under runtime: [#2843](https://github.com/owncloud/ocis/pull/2843)
-* Enhancement - Update REVA to v1.17.0: [#2835](https://github.com/owncloud/ocis/pull/2835)
+* Enhancement - Update REVA to v1.17.0: [#2849](https://github.com/owncloud/ocis/pull/2849)
+* Enhancement - Update ownCloud Web to v4.6.1: [#2846](https://github.com/owncloud/ocis/pull/2846)
## Details
@@ -46,6 +98,15 @@ The following sections list the changes for unreleased.
https://github.com/owncloud/ocis/issues/2668
https://github.com/owncloud/ocis/pull/2807
+* Bugfix - Use the CS3api up- and download workflow for the accounts service: [#2837](https://github.com/owncloud/ocis/pull/2837)
+
+ We've fixed the interaction of the accounts service with the metadata storage after bypassing
+ the InitiateUpload and InitiateDownload have been removed from various storage drivers. The
+ accounts service now uses the proper CS3apis workflow for up- and downloads.
+
+ https://github.com/owncloud/ocis/pull/2837
+ https://github.com/cs3org/reva/pull/2309
+
* Change - Rename `APP_PROVIDER_BASIC_*` environment variables: [#2812](https://github.com/owncloud/ocis/pull/2812)
We've renamed the `APP_PROVIDER_BASIC_*` to `APP_PROVIDER_*` since the `_BASIC_` part is a
@@ -82,13 +143,38 @@ The following sections list the changes for unreleased.
https://github.com/owncloud/ocis/pull/2843
-* Enhancement - Update REVA to v1.17.0: [#2835](https://github.com/owncloud/ocis/pull/2835)
+* Enhancement - Update REVA to v1.17.0: [#2849](https://github.com/owncloud/ocis/pull/2849)
Updated REVA to v1.17.0 This update includes:
- * #TODO: update this changelog before the next oCIS release
+ * Fix [cs3org/reva#2305](https://github.com/cs3org/reva/pull/2305): Make sure /app/new takes `target` as absolute path
+ * Fix [cs3org/reva#2303](https://github.com/cs3org/reva/pull/2303): Fix content disposition header for public links files
+ * Fix [cs3org/reva#2316](https://github.com/cs3org/reva/pull/2316): Fix the share types in propfinds
+ * Fix [cs3org/reva#2803](https://github.com/cs3org/reva/pull/2310): Fix app provider for editor public links
+ * Fix [cs3org/reva#2298](https://github.com/cs3org/reva/pull/2298): Remove share refs from trashbin
+ * Fix [cs3org/reva#2309](https://github.com/cs3org/reva/pull/2309): Remove early finish for zero byte file uploads
+ * Fix [cs3org/reva#1941](https://github.com/cs3org/reva/pull/1941): Fix TUS uploads with transfer token only
+ * Chg [cs3org/reva#2210](https://github.com/cs3org/reva/pull/2210): Fix app provider new file creation and improved error codes
+ * Enh [cs3org/reva#2217](https://github.com/cs3org/reva/pull/2217): OIDC auth driver for ESCAPE IAM
+ * Enh [cs3org/reva#2256](https://github.com/cs3org/reva/pull/2256): Return user type in the response of the ocs GET user call
+ * Enh [cs3org/reva#2315](https://github.com/cs3org/reva/pull/2315): Add new attributes to public link propfinds
+ * Enh [cs3org/reva#2740](https://github.com/cs3org/reva/pull/2250): Implement space membership endpoints
+ * Enh [cs3org/reva#2252](https://github.com/cs3org/reva/pull/2252): Add the xattr sys.acl to SysACL (eosgrpc)
+ * Enh [cs3org/reva#2314](https://github.com/cs3org/reva/pull/2314): OIDC: fallback if IDP doesn't provide "preferred_username" claim
+ https://github.com/owncloud/ocis/pull/2849
https://github.com/owncloud/ocis/pull/2835
+ https://github.com/owncloud/ocis/pull/2837
+
+* Enhancement - Update ownCloud Web to v4.6.1: [#2846](https://github.com/owncloud/ocis/pull/2846)
+
+ Tags: web
+
+ We updated ownCloud Web to v4.6.1. Please refer to the changelog (linked) for details on the web
+ release.
+
+ https://github.com/owncloud/ocis/pull/2846
+ https://github.com/owncloud/web/releases/tag/v4.6.1
# Changelog for [1.15.0] (2021-11-19)
The following sections list the changes for 1.15.0.
diff --git a/changelog/unreleased/change-rename-app-provider-env.md b/changelog/1.16.0_2021-12-10/change-rename-app-provider-env.md
similarity index 100%
rename from changelog/unreleased/change-rename-app-provider-env.md
rename to changelog/1.16.0_2021-12-10/change-rename-app-provider-env.md
diff --git a/changelog/unreleased/fix-basic-auth-route-claim-selector.md b/changelog/1.16.0_2021-12-10/fix-basic-auth-route-claim-selector.md
similarity index 100%
rename from changelog/unreleased/fix-basic-auth-route-claim-selector.md
rename to changelog/1.16.0_2021-12-10/fix-basic-auth-route-claim-selector.md
diff --git a/changelog/unreleased/fix-create-group-without-name.md b/changelog/1.16.0_2021-12-10/fix-create-group-without-name.md
similarity index 100%
rename from changelog/unreleased/fix-create-group-without-name.md
rename to changelog/1.16.0_2021-12-10/fix-create-group-without-name.md
diff --git a/changelog/unreleased/fix-s3ng-metadata-storage.md b/changelog/1.16.0_2021-12-10/fix-s3ng-metadata-storage.md
similarity index 100%
rename from changelog/unreleased/fix-s3ng-metadata-storage.md
rename to changelog/1.16.0_2021-12-10/fix-s3ng-metadata-storage.md
diff --git a/changelog/unreleased/fix-use-cs3apis-up-and-download-workflow-accounts.md b/changelog/1.16.0_2021-12-10/fix-use-cs3apis-up-and-download-workflow-accounts.md
similarity index 100%
rename from changelog/unreleased/fix-use-cs3apis-up-and-download-workflow-accounts.md
rename to changelog/1.16.0_2021-12-10/fix-use-cs3apis-up-and-download-workflow-accounts.md
diff --git a/changelog/unreleased/ocis-pkg-config-cleanup.md b/changelog/1.16.0_2021-12-10/ocis-pkg-config-cleanup.md
similarity index 100%
rename from changelog/unreleased/ocis-pkg-config-cleanup.md
rename to changelog/1.16.0_2021-12-10/ocis-pkg-config-cleanup.md
diff --git a/changelog/unreleased/revamp-config.md b/changelog/1.16.0_2021-12-10/revamp-config.md
similarity index 100%
rename from changelog/unreleased/revamp-config.md
rename to changelog/1.16.0_2021-12-10/revamp-config.md
diff --git a/changelog/unreleased/supervisor-stop-routine.md b/changelog/1.16.0_2021-12-10/supervisor-stop-routine.md
similarity index 100%
rename from changelog/unreleased/supervisor-stop-routine.md
rename to changelog/1.16.0_2021-12-10/supervisor-stop-routine.md
diff --git a/changelog/1.16.0_2021-12-10/update-reva.md b/changelog/1.16.0_2021-12-10/update-reva.md
new file mode 100644
index 0000000000..7f5b4753bf
--- /dev/null
+++ b/changelog/1.16.0_2021-12-10/update-reva.md
@@ -0,0 +1,24 @@
+Enhancement: Update REVA to v1.17.0
+
+Updated REVA to v1.17.0
+This update includes:
+
+ * Fix [cs3org/reva#2305](https://github.com/cs3org/reva/pull/2305): Make sure /app/new takes `target` as absolute path
+ * Fix [cs3org/reva#2303](https://github.com/cs3org/reva/pull/2303): Fix content disposition header for public links files
+ * Fix [cs3org/reva#2316](https://github.com/cs3org/reva/pull/2316): Fix the share types in propfinds
+ * Fix [cs3org/reva#2803](https://github.com/cs3org/reva/pull/2310): Fix app provider for editor public links
+ * Fix [cs3org/reva#2298](https://github.com/cs3org/reva/pull/2298): Remove share refs from trashbin
+ * Fix [cs3org/reva#2309](https://github.com/cs3org/reva/pull/2309): Remove early finish for zero byte file uploads
+ * Fix [cs3org/reva#1941](https://github.com/cs3org/reva/pull/1941): Fix TUS uploads with transfer token only
+ * Chg [cs3org/reva#2210](https://github.com/cs3org/reva/pull/2210): Fix app provider new file creation and improved error codes
+ * Enh [cs3org/reva#2217](https://github.com/cs3org/reva/pull/2217): OIDC auth driver for ESCAPE IAM
+ * Enh [cs3org/reva#2256](https://github.com/cs3org/reva/pull/2256): Return user type in the response of the ocs GET user call
+ * Enh [cs3org/reva#2315](https://github.com/cs3org/reva/pull/2315): Add new attributes to public link propfinds
+ * Enh [cs3org/reva#2740](https://github.com/cs3org/reva/pull/2250): Implement space membership endpoints
+ * Enh [cs3org/reva#2252](https://github.com/cs3org/reva/pull/2252): Add the xattr sys.acl to SysACL (eosgrpc)
+ * Enh [cs3org/reva#2314](https://github.com/cs3org/reva/pull/2314): OIDC: fallback if IDP doesn't provide "preferred_username" claim
+
+
+https://github.com/owncloud/ocis/pull/2849
+https://github.com/owncloud/ocis/pull/2835
+https://github.com/owncloud/ocis/pull/2837
diff --git a/changelog/1.16.0_2021-12-10/update-web-4.6.1.md b/changelog/1.16.0_2021-12-10/update-web-4.6.1.md
new file mode 100644
index 0000000000..55b909510c
--- /dev/null
+++ b/changelog/1.16.0_2021-12-10/update-web-4.6.1.md
@@ -0,0 +1,8 @@
+Enhancement: Update ownCloud Web to v4.6.1
+
+Tags: web
+
+We updated ownCloud Web to v4.6.1. Please refer to the changelog (linked) for details on the web release.
+
+https://github.com/owncloud/ocis/pull/2846
+https://github.com/owncloud/web/releases/tag/v4.6.1
diff --git a/changelog/unreleased/user-claim-fallback.md b/changelog/1.16.0_2021-12-10/user-claim-fallback.md
similarity index 100%
rename from changelog/unreleased/user-claim-fallback.md
rename to changelog/1.16.0_2021-12-10/user-claim-fallback.md
diff --git a/changelog/unreleased/enhancement-add-new-file-capability.md b/changelog/unreleased/enhancement-add-new-file-capability.md
new file mode 100644
index 0000000000..57fe8c5b5e
--- /dev/null
+++ b/changelog/unreleased/enhancement-add-new-file-capability.md
@@ -0,0 +1,8 @@
+Enhancement: add new file url of the app provider to the ocs capabilities
+
+We've added the new file capability of the app provider to the ocs capabilities, so that
+clients can discover this url analogous to the app list and file open urls.
+
+https://github.com/owncloud/ocis/pull/2884
+https://github.com/cs3org/reva/pull/2379
+https://github.com/owncloud/web/pull/5890#issuecomment-993905242
diff --git a/changelog/unreleased/public-link-signature-auth.md b/changelog/unreleased/public-link-signature-auth.md
new file mode 100644
index 0000000000..fc50ca8657
--- /dev/null
+++ b/changelog/unreleased/public-link-signature-auth.md
@@ -0,0 +1,5 @@
+Enhancement: Support signature auth in the public share auth middleware
+
+Enabled public share requests to be authenticated using the public share signature.
+
+https://github.com/owncloud/ocis/pull/2831
diff --git a/changelog/unreleased/update-libregraph-api.md b/changelog/unreleased/update-libregraph-api.md
new file mode 100644
index 0000000000..f8da54c775
--- /dev/null
+++ b/changelog/unreleased/update-libregraph-api.md
@@ -0,0 +1,5 @@
+Change: Update libre-graph-api to v0.3.0
+
+This updates the libre-graph-api to use the latest spec and types.
+
+https://github.com/owncloud/ocis/pull/2858
diff --git a/changelog/unreleased/update-reva.md b/changelog/unreleased/update-reva.md
index 810cb656a8..d5924214b4 100644
--- a/changelog/unreleased/update-reva.md
+++ b/changelog/unreleased/update-reva.md
@@ -1,10 +1,8 @@
-Enhancement: Update REVA to v1.17.0
+Enhancement: Update REVA to xxx
-Updated REVA to v1.17.0
+Updated REVA to xxx
This update includes:
- * #TODO: update this changelog before the next oCIS release
+ * TODO: update changelog before oCIS release
-
-https://github.com/owncloud/ocis/pull/2835
-https://github.com/owncloud/ocis/pull/2837
+https://github.com/owncloud/ocis/pull/2878
\ No newline at end of file
diff --git a/changelog/unreleased/update-spaces-handle-non-existent.md b/changelog/unreleased/update-spaces-handle-non-existent.md
new file mode 100644
index 0000000000..c69fbc8d25
--- /dev/null
+++ b/changelog/unreleased/update-spaces-handle-non-existent.md
@@ -0,0 +1,5 @@
+Change: Return not found when updating non existent space
+
+If a spaceid of a space which is updated doesn't exist, handle it as a not found error.
+
+https://github.com/cs3org/reva/pull/2869
diff --git a/docs/extensions/graph/spaces.md b/docs/extensions/graph/spaces.md
index 19ffa3f4c5..f20db0e2e5 100644
--- a/docs/extensions/graph/spaces.md
+++ b/docs/extensions/graph/spaces.md
@@ -130,6 +130,7 @@ curl -k 'https://localhost:9200/graph/v1.0/me/drives' -u einstein:relativity -v
}
}
]
+```
As we can see the response already contains a space-aware dav endpoint, which we can use to upload files to the space:
diff --git a/docs/extensions/storage/_index.md b/docs/extensions/storage/_index.md
index d166e6005f..8ae194026c 100644
--- a/docs/extensions/storage/_index.md
+++ b/docs/extensions/storage/_index.md
@@ -8,16 +8,13 @@ geekdocFilePath: _index.md
geekdocCollapseSection: true
---
-## Abstract
+## Overview
-This service provides an oCIS extension that wraps [reva](https://github.com/cs3org/reva/) and adds an opinionated configuration to it.
+The storage extension wraps [reva](https://github.com/cs3org/reva/) and adds an opinionated configuration to provide two core services for the oCIS platform:
+1. A [*Spaces Registry*]({{< ref "./spacesregistry.md" >}}) that acts as a dictionary for storage *Spaces* and their metadata
+2. A [*Spaces Provider*]({{< ref "./spacesprovider.md" >}}) that organizes *Resources* in storage *Spaces* and persists them in an underlying *Storage System*
-## Architecture Overview
-
-The below diagram shows the oCIS services and the contained reva services within as dashed boxes. In general:
-1. A request comes in at the proxy and is authenticated using OIDC.
-2. It is forwarded to the oCIS frontend which handles ocs and ocdav requests by talking to the reva gateway using the CS3 API.
-3. The gateway acts as a facade to the actual CS3 services: storage providers, user providers, group providers and sharing providers.
+*Clients* will use the *Spaces Registry* to poll or get notified about changes in all *Spaces* a user has access to. Every *Space* has a dedicated `/dav/spaces/` WebDAV endpoint that is served by a *Spaces Provider* which uses a specific reva storage driver to wrap an underlying *Storage System*.
{{< svg src="extensions/storage/static/overview.drawio.svg" >}}
@@ -34,3 +31,4 @@ The bottom part is lighter because we will deprecate it in favor of using only t
In order to reason about the request flow, two aspects in the architecture need to be understood well:
1. What kind of [*namespaces*]({{< ref "./namespaces.md" >}}) are presented at the different WebDAV and CS3 endpoints?
2. What kind of [*resource*]({{< ref "./terminology.md#resources" >}}) [*references*]({{< ref "./terminology.md#references" >}}) are exposed or required: path or id based?
+{{< svg src="extensions/storage/static/storage.drawio.svg" >}}
diff --git a/docs/extensions/storage/apps.md b/docs/extensions/storage/apps.md
index 19826cf485..1ca902ba69 100644
--- a/docs/extensions/storage/apps.md
+++ b/docs/extensions/storage/apps.md
@@ -24,7 +24,8 @@ The capabilities endpoint (eg. `https://localhost:9200/ocs/v1.php/cloud/capabili
"enabled": true,
"version": "1.0.0",
"apps_url": "/app/list",
- "open_url": "/app/open"
+ "open_url": "/app/open",
+ "new_url": "/app/new"
}
]
}
@@ -101,6 +102,8 @@ curl 'https://ocis.test/app/list'
**Response example**:
+HTTP status code: 200
+
```json
{
"mime-types": [
@@ -132,7 +135,8 @@ curl 'https://ocis.test/app/list'
"name": "OpenDocument",
"icon": "https://some-website.test/opendocument-text-icon.png",
"description": "OpenDocument text document",
- "allow_creation": true
+ "allow_creation": true,
+ "default_application": "Collabora"
},
{
"mime_type": "text/markdown",
@@ -145,7 +149,8 @@ curl 'https://ocis.test/app/list'
],
"name": "Markdown file",
"description": "Markdown file",
- "allow_creation": true
+ "allow_creation": true,
+ "default_application": "CodiMD"
},
{
"mime_type": "application/vnd.ms-word.document.macroenabled.12",
@@ -216,6 +221,8 @@ All apps are expected to be opened in an iframe and the response will give some
There are apps, which need to be opened in the iframe with a form post. The form post must include all form parameters included in the response. For these apps the response will look like this:
+HTTP status code: 200
+
```json
{
"app_url": "https://.....",
@@ -230,6 +237,8 @@ There are apps, which need to be opened in the iframe with a form post. The form
There are apps, which need to be opened in the iframe with a GET request. The GET request must have set all headers included in the response. For these apps the response will look like this:
+HTTP status code: 200
+
```json
{
"app_url": "https://...",
@@ -244,33 +253,175 @@ There are apps, which need to be opened in the iframe with a GET request. The GE
**Example responses (error case)**:
-- wrong `view_mode`
+- missing `file_id`
+
+ HTTP status code: 400
```json
{
- "code": "SERVER_ERROR",
- "message": "Missing or invalid viewmode argument"
+ "code": "INVALID_PARAMETER",
+ "message": "missing file ID"
+ }
+ ```
+
+- wrong `view_mode`
+
+ HTTP status code: 400
+
+ ```json
+ {
+ "code": "INVALID_PARAMETER",
+ "message": "invalid view mode"
}
```
- unknown `app_name`
- ```json
- {
- "code": "SERVER_ERROR",
- "message": "error searching for app provider"
- }
- ```
-
-- wrong / invalid file id / unauthorized to open the file
+ HTTP status code: 404
```json
{
- "code": "SERVER_ERROR",
- "message": "error statting file"
+ "code": "RESOURCE_NOT_FOUND",
+ "message": "error: not found: app 'Collabor' not found"
}
```
+- wrong / invalid file id
+
+ HTTP status code: 400
+
+ ```json
+ {
+ "code": "INVALID_PARAMETER",
+ "message": "invalid file ID"
+ }
+ ```
+
+- file id does not point to a file
+
+ HTTP status code: 400
+
+ ```json
+ {
+ "code": "INVALID_PARAMETER",
+ "message": "the given file id does not point to a file"
+ }
+ ```
+
+- file does not exist / unauthorized to open the file
+
+ HTTP status code: 404
+
+ ```json
+ {
+ "code": "RESOURCE_NOT_FOUND",
+ "message": "file does not exist"
+ }
+ ```
+
+### Creating a file with the app provider
+
+**Endpoint**: specified in the capabilities in `new_file_url`, currently `/app/new`
+
+**Method**: HTTP POST
+
+**Authentication** (one of them):
+
+- `Authorization` header with OIDC Bearer token for authenticated users or basic auth credentials (if enabled in oCIS)
+- `Public-Token` header with public link token for public links
+- `X-Access-Token` header with a REVA token for authenticated users
+
+**Query parameters**:
+
+- `parent_container_id` (mandatory): ID of the folder in which the file will be created
+- `filename` (mandatory): name of the new file
+- `template` (optional): not yet implemented
+
+**Request examples**:
+
+```bash
+curl -X POST 'https://ocis.test/app/new?parent_container_id=c2lkOmNpZAo=&filename=test.odt'
+```
+
+**Response example**:
+
+You will receive a file id of the freshly created file, which you can use to open the file in an editor.
+
+```json
+{
+ "file_id": "ZmlsZTppZAo="
+}
+```
+
+**Example responses (error case)**:
+
+- missing `parent_container_id`
+
+ HTTP status code: 400
+
+ ```json
+ {
+ "code": "INVALID_PARAMETER",
+ "message": "missing parent container ID"
+ }
+ ```
+
+- missing `filename`
+
+ HTTP status code: 400
+
+ ```json
+ {
+ "code": "INVALID_PARAMETER",
+ "message": "missing filename"
+ }
+ ```
+
+- parent container not found
+
+ HTTP status code: 404
+
+ ```json
+ {
+ "code": "RESOURCE_NOT_FOUND",
+ "message": "the parent container is not accessible or does not exist"
+ }
+ ```
+
+- `parent_container_id` does not point to a container
+
+ HTTP status code: 400
+
+ ```json
+ {
+ "code": "INVALID_PARAMETER",
+ "message": "the parent container id does not point to a container"
+ }
+ ```
+
+- `filename` is invalid (eg. includes a path segment)
+
+ HTTP status code: 400
+
+ ```json
+ {
+ "code": "INVALID_PARAMETER",
+ "message": "the filename must not contain a path segment"
+ }
+ ```
+
+- file already exists
+
+ HTTP status code: 403
+
+ ```json
+ {
+ "code": "RESOURCE_ALREADY_EXISTS",
+ "message": "the file already exists"
+ }
+ ```
+
+
## App drivers
App drivers represent apps, if the app is not able to register itself. Currently there is only the CS3org WOPI server app driver.
diff --git a/docs/extensions/storage/namespaces.md b/docs/extensions/storage/namespaces.md
index 3ca0f9981d..19864a6e7f 100644
--- a/docs/extensions/storage/namespaces.md
+++ b/docs/extensions/storage/namespaces.md
@@ -12,11 +12,11 @@ In ownCloud 10 all paths are considered relative to the users home. The CS3 API
{{< svg src="extensions/storage/static/namespaces.drawio.svg" >}}
-The different paths in the namespaces need to be translated while passing [*references*]({{< ref "./terminology.md#references" >}}) from service to service. While the oc10 endpoints all work on paths we internally reference shared resources by id, so the shares don't break when a file is renamed or moved inside a [*storage space*]({{< ref "./terminology.md#storage-spaces" >}}). The following table lists the various namespaces, paths and id based references:
+The different paths in the namespaces need to be translated while passing [*references*]({{< ref "./terminology.md#references" >}}) from service to service. While the oc10 endpoints all work on paths we internally reference shared resources by id, so the shares don't break when a file is renamed or moved inside a storage [*space*]({{< ref "./spaces" >}}). The following table lists the various namespaces, paths and id based references:
| oc10 namespace | CS3 global namespace | storage provider | reference | content |
|--------------------------------------------------|----------------------------------------|------------------|-----------|---------|
-| `/webdav/path/to/file.ext` `/dav/files//path/to/file.ext` | `/home/path/to/file.ext` | home | `//path/to/file.ext` | currently logged in users home |
+| `/webdav/path/to/file.ext` `/dav/files//path/to/file.ext` | `/home/path/to/file.ext` | home | `//path/to/file.ext` | currently logged in users home |
| `/webdav/Shares/foo` `/dav/files//Shares/foo` | `/home/Shares/foo` | users | id based access | all users, used to access collaborative shares |
| `/dav/public-files//rel/path/to/file.ext` | `/public//rel/path/to/file.ext` | public | id based access | publicly shared files, used to access public links |
@@ -32,26 +32,26 @@ In the global CS3 namespaces we plan to move `/home/Shares`, which currently lis
## ownCloud namespaces
-In contrast to the global namespace of CS3, ownCloud always presented a user specific namespace on all endpoints. It will always list the users private files under `/`. Shares can be mounted at an arbitrary location in the users private spaces. See the [webdav]({{< ref "./architecture#webdav" >}}) and [ocs]({{< ref "./architecture#sharing" >}}) sections for more details end examples.
+In contrast to the global namespace of CS3, ownCloud always presented a user specific namespace on all endpoints. It will always list the users private files under `/`. Shares can be mounted at an arbitrary location in the users private spaces. See the [webdav]({{< ref "./spacesprovider#webdav" >}}) and [ocs]({{< ref "./spacesprovider#sharing" >}}) sections for more details end examples.
With the spaces concept we are planning to introduce a global namespace to the ownCloud webdav endpoints. This will push the users private space down in the hierarchy: it will move from `/webdav` to `/webdav/home` or `/webdav/users/`. The related [migration stages]({{< ref "../../ocis/migration.md" >}}) are subject to change.
## CS3 global namespaces
-The *CS3 global namespace* in oCIS is configured in the [*storage space registry*]({{< ref "./terminology.md#storage-space-registries" >}}). oCIS uses these defaults:
+The *CS3 global namespace* in oCIS is configured in the storage [*spaces registry*]({{< ref "./spacesregistry" >}}). oCIS uses these defaults:
| global namespace | description |
|-|-|
| `/home` | an alias for the currently logged in uses private space |
-| `/users/` | user private spaces |
+| `/users/` | user private spaces |
| `/shares` | a virtual listing of share spaces a user has access to |
| `/public/` | a virtual folder listing public shares |
| `/spaces/` | *TODO: project or group spaces* |
-Technically, the `/home` namespace is not necessary: the [*storage space registry*]({{< ref "./terminology.md#storage-space-registries" >}}) knows the path to a users private space in the `/users` namespace and the gateway can forward the requests to the responsible storage provider.
+Technically, the `/home` namespace is not necessary: the storage [*spaces registry*]({{< ref "./spacesregistry" >}}) knows the path to a users private space in the `/users` namespace and the gateway can forward the requests to the responsible storage provider.
{{< hint warning >}}
-*@jfd: Why don't we use `/home/` instead of `/users/`. Then the paths would be consistent with most unix systems.
+*@jfd: Why don't we use `/home/` instead of `/users/`. Then the paths would be consistent with most unix systems.
{{< /hint >}}
The `/shares` namespace is used to solve two problems:
diff --git a/docs/extensions/storage/proposedchanges.md b/docs/extensions/storage/proposedchanges.md
new file mode 100644
index 0000000000..c306a4005b
--- /dev/null
+++ b/docs/extensions/storage/proposedchanges.md
@@ -0,0 +1,177 @@
+---
+title: "Proposed Changes"
+date: 2018-05-02T00:00:00+00:00
+weight: 18
+geekdocRepo: https://github.com/owncloud/ocis
+geekdocEditPath: edit/master/docs/extensions/storage
+geekdocFilePath: proposedchanges.md
+---
+
+Some architectural changes still need to be clarified or changed. Maybe an ADR is in order for all of the below.
+
+## Reva Gateway changes
+
+## A dedicated shares storage provider
+
+Currently, when a user accepts a share, a cs3 reference is created in the users `/home/shares` folder. This reference represents the mount point of a share and can be renamed, similar to the share jail in ownCloud 10. This spreads the metadata of a share in two places:
+- the share is persisted in the *share manager*
+- the mount point of a share is persisted in the home *storage provider*
+
+Furthermore, the *gateway* treats `/home/shares` different than any other path: it will stat all children and calculate an etag to allow clients to discover changes in accepted shares. This requires the storage provider to cooperate and provide this special `/shares` folder in the root of a users home when it is accessed as a home storage. That is the origin of the `enable_home` config flag that needs to be implemented for every storage driver.
+
+In order to have a single source of truth we need to make the *share manager* aware of the mount point. We can then move all the logic that aggregates the etag in the share folder to a dedicated *shares storage provider* that is using the *share manager* for persistence. The *shares storage provider* would provide a `/shares` namespace outside of `/home` that lists all accepted shares for the current user. As a result the storage drivers no longer need to have a `enable_home` flag that jails users into their home. The `/home/shares` folder would move outside of the `/home`. In fact `/home` will no longer be needed, because the home folder concept can be implemented as a space: `CreateHome` would create a `personal` space on the.
+
+Work on this is done in https://github.com/cs3org/reva/pull/2023
+
+{{< hint warning >}}
+What about copy pasting links from the browser? Well this storage is only really needed to have a path to ocm shares that actually reside on other instances. In the UI the shares would be listed by querying a *share manager*. It returns ResourceIds, which can be stated to fetch a path that is then accessible in the CS3 global namespace. Two caveats:
+- This only works for resources that are actually hosted by the current instance. For those it would leak the parent path segments to a shared resource.
+- For accepted OCM shares there must be a path in the [*CS3 global namespace*]({{< ref "./namespaces.md#cs3-global-namespaces" >}}) that has to be the same for all users, otherwise they cannot copy and share those URLs.
+{{< /hint >}}
+
+### The gateway should be responsible for path transformations
+
+Currently, storage providers are aware af their mount point, coupling them tightly with the gateway.
+
+Tracked in https://github.com/cs3org/reva/issues/578
+
+Work is done in https://github.com/cs3org/reva/pull/1866
+
+## URL escaped string representation of a CS3 reference
+
+For the spaces concept we introduced the `/dav/spaces/` endpoint. It encodes a cs3 *reference* in a URL compatible way.
+1. We can separate the path using a `/`: `/dav/spaces//`
+2. The `spaceid` currently is a cs3 resourceid, consisting of `` and ``. Since the opaqueid might contain `/` eg. for the local driver we have to urlencode the spaceid.
+
+To access resources by id we need to make the `/dav/meta/` able to list directories... Otherwise id based navigation first has to look up the path. Or we use the libregraph api for id based navigation.
+
+A *reference* is a logical concept. It identifies a [*resource*]({{< ref "#resources" >}}) and consists of a `` and a ``. A `` consists of a `` and a ``. They can be concatenated using the separators `!` and `:`:
+```
+!:
+```
+While all components are optional, only three cases are used:
+| format | example | description |
+|-|-|-|
+| `!:` | `!:/absolute/path/to/file.ext` | absolute path |
+| `!:` | `ee1687e5-ac7f-426d-a6c0-03fed91d5f62!:path/to/file.ext` | path relative to the root of the storage space |
+| `!:` | `ee1687e5-ac7f-426d-a6c0-03fed91d5f62!c3cf23bb-8f47-4719-a150-1d25a1f6fb56:to/file.ext` | path relative to the specified node in the storage space, used to reference resources without disclosing parent paths |
+
+`` should be a UUID to prevent references from breaking when a *user* or [*storage space*]({{< ref "#storage-spaces" >}}) gets renamed. But it can also be derived from a migration of an oc10 instance by concatenating an instance identifier and the numeric storage id from oc10, e.g. `oc10-instance-a$1234`.
+
+A reference will often start as an absolute/global path, e.g. `!:/home/Projects/Foo`. The gateway will look up the storage provider that is responsible for the path
+
+| Name | Description | Who resolves it? |
+|------|-------------|-|
+| `!:/home/Projects/Foo` | the absolute path a client like davfs will use. | The gateway uses the storage registry to look up the responsible storage provider |
+| `ee1687e5-ac7f-426d-a6c0-03fed91d5f62!:/Projects/Foo` | the `storage_space` is the same as the `root`, the path becomes relative to the root | the storage provider can use this reference to identify this resource |
+
+Now, the same file is accessed as a share
+| Name | Description |
+|------|-------------|
+| `!:/users/Einstein/Projects/Foo` | `Foo` is the shared folder |
+| `ee1687e5-ac7f-426d-a6c0-03fed91d5f62!56f7ceca-e7f8-4530-9a7a-fe4b7ec8089a:` | `56f7ceca-e7f8-4530-9a7a-fe4b7ec8089a` is the id of `Foo`, the path is empty |
+
+
+The `:`, `!` and `$` are chosen from the set of [RFC3986 sub delimiters](https://tools.ietf.org/html/rfc3986#section-2.2) on purpose. They can be used in URLs without having to be encoded. In some cases, a delimiter can be left out if a component is not set:
+| reference | interpretation |
+|-|-|
+| `/absolute/path/to/file.ext` | absolute path, all delimiters omitted |
+| `ee1687e5-ac7f-426d-a6c0-03fed91d5f62!path/to/file.ext` | relative path in the given storage space, root delimiter `:` omitted |
+| `56f7ceca-e7f8-4530-9a7a-fe4b7ec8089a:to/file.ext` | relative path in the given root node, storage space delimiter `!` omitted |
+| `ee1687e5-ac7f-426d-a6c0-03fed91d5f62!56f7ceca-e7f8-4530-9a7a-fe4b7ec8089a:` | node id in the given storage space, `:` must be present |
+| `ee1687e5-ac7f-426d-a6c0-03fed91d5f62` | root of the storage space, all delimiters omitted, can be distinguished by the `/` |
+
+## space providers
+When looking up an id based resource the reference must use a logical space id, not a CS3 resource id. Otherwise id based requests, which only have a resourceid consisting of a storage id and a node id cannot be routed to the correct storage provider if the storage has moved from one storage provider to another.
+
+if the registry routes based on the storageid AND the nodeid it has to keep a cache of all nodeids in order to route all requests for a storage space (which consists of storage it + nodeid) to the correct storage provider. the correct resourceid for a node in a storage space would be `$!`. The `$` part allow the storage registry to route all id based requests to the correct storage provider. This becomes relevant when the storage space was moved from one storage provider to another. The storage space id remains the same, but the internal address and port change.
+
+TODO discuss to clarify further
+
+## Storage drivers
+
+### allow clients to send a uuid on upload
+iOS clients can only queue single requests to be executed in the background. They queue an upload and need to be able to identify the uploaded file after it has been uploaded to the server. The disconnected nature of the connection might cause workflows or manual user interaction with the file on the server to move the file to a different place or changing the content while the device is offline. However, on the device users might have marked the file as favorite or added it to other iOS specific collections. To be able to reliably identify the file the client can generate a `uuid` and attach it to the file metadata during the upload. While it is not necessary to look up files by this `uuid` having a second file id that serves exactly the same purpose as the `file id` is redundant.
+
+Another aspect for the `file id` / `uuid` is that it must be a logical identifier that can be set, at least by internal systems. Without a writeable fileid we cannot restore backups or migrate storage spaces from one storage provider to another storage provider.
+
+Technically, this means that every storage driver needs to have a map of a `uuid` to an internal resource identifier. This internal resource identifier can be
+- an eos fileid, because eos can look up files by id
+- an inode if the filesystem and the storage driver support looking up by inode
+- a path if the storage driver has no way of looking up files by id.
+ - In this case other mechanisms like inotify, kernel audit or a fuse overlay might be used to keep the paths up to date.
+ - to prevent excessive writes when deep folders are renamed a reverse map might be used: it will map the `uuid` to `:`, in order to trade writes for reads
+ - as a fallback a sync job can read the file id from the metadata of the resources and populate the uuid to internal id map.
+
+The TUS upload can take metadata, for PUT we might need a header.
+
+### Space id vs resource id vs storage id
+
+We have `/dav/meta/` where the `fileid` is a string that was returned by a PROPFIND or by the `/graph/v1.0/me/drives/` endpoint? That returns a space id and the root drive item which has an `id`
+
+Does that `id` have a specific format? We currently concatenate as `!`.
+
+A request against `/dav/meta/fileid` will use the reva storage registry to look up a path.
+
+What if the storage space is moved to another storage provider. This happens during a migration:
+
+1. the current oc10 fileids need to be prefixed with at least the numeric storage id to shard them.
+
+`123` becomes `instanceprefix$345!123` if we use a custom prefix that identifies an instance (so we can merge multiple instances into one ocis instance) and append the numeric storageid `345`. The pattern is `$!`.
+
+Every `$` identifies a space.
+
+- [ ] the owncloudsql driver can return these spaceids when listing spaces.
+
+Why does it not work if we just use the fileid of the root node in the db?
+
+Say we have a space with three resources:
+`$!`
+`instanceprefix$345!1`
+`instanceprefix$345!2`
+`instanceprefix$345!3`
+
+All users have moved to ocis and the registry contains a regex to route all `instanceprefix.*` references to the storageprovider with the owncloudsql driver. It is up to the driver to locate the correct resource by using the filecache table. In this case the numeric storage id is unnecessary.
+
+Now we migrate the space `345` to another storage driver:
+- the storage registry contains a new entry for `instanceprefix$345` to send all resource ids for that space to the new storage provider
+- the new storage driver has to take into account the full storageid because the nodeid may only be unique per storage space.
+
+If we now have to fetch the path on the `/dav/meta/` endpoint:
+`/dav/meta/instanceprefix$345!1`
+`/dav/meta/instanceprefix$345!2`
+`/dav/meta/instanceprefix$345!3`
+
+This would work because the registry always sees `instanceprefix$345` as the storageid.
+
+Now if we use the fileids directly and leave out the numeric storageid:
+`!`
+`instanceprefix!1`
+`instanceprefix!2`
+`instanceprefix!3`
+
+This is the current `!` format.
+
+The reva storage registry contains a `instanceid` entry pointing to the storage provider with the owncloudsql driver.
+
+Resources can be looked up because the oc_filecache has a unique fileid over all storages.
+
+Now we again migrate the space `345` to another storage driver:
+- the storage registry contains a new entry for `instanceprefix!1` so the storage space root now points to the new storage provider
+- The registry needs to be aware of node ids to route properly. This is a no go. We don't want to keep a cache of *all* nodeids in the registry. Only the root nodes of spaces.
+- The new storage driver only has a nodeid which might collide with other nodeids from other storage spaces, eg when two instances are imported into one ocis instance. Although it would be possible to just set up two storage providers extra care would have to be taken to prevent nodeid collisions when importing a space.
+
+If we now have to fetch the path on the `/dav/meta/` endpoint:
+`/dav/meta/instanceprefix!1` would work because it is the root of a space
+`/dav/meta/instanceprefix!2` would cause the gateway to poll all storage providers because the registry has no way to determine the responsible storage provider
+`/dav/meta/instanceprefix!3` same
+
+The problem is that without a part in the storageid that allows differentiating storage spaces we cannot route them individually.
+
+Now, we could use the nodeid of the root of a storage space as the spaceid ... if it is a uuid. If it is numeric it needs a prefix to distinguish it from other spaces.
+`!` would be easy for the decomposedfs.
+eos might use numeric ids: `$!`, but it needs a custom prefix to distinguish multiple eos instances.
+
+Furthermore, when migrating spaces between storage providers we want to stay collision free, which is why we should recommend uuids.
+
+All this has implications for the decomposedfs, because it needs to split the nodes per space to prevent them from colliding.
diff --git a/docs/extensions/storage/releasing.md b/docs/extensions/storage/releasing.md
deleted file mode 100644
index b55369d32e..0000000000
--- a/docs/extensions/storage/releasing.md
+++ /dev/null
@@ -1,31 +0,0 @@
----
-title: "Releasing"
-date: 2020-05-22T00:00:00+00:00
-weight: 60
-geekdocRepo: https://github.com/owncloud/ocis
-geekdocEditPath: edit/master/docs/extensions/storage
-geekdocFilePath: releasing.md
----
-
-{{< toc >}}
-
-To release a new version of the storage submodule, you have to follow a few simple steps.
-
-## Preparation
-
-1. Before releasing, make sure that reva has been [updated to the desired version]({{< ref "updating" >}})
-
-## Release
-1. Check out master
-{{< highlight txt >}}
-git checkout master
-git pull origin master
-{{< / highlight >}}
-2. Create a new tag (preferably signed) and replace the version number accordingly. Prefix the tag with the submodule `storage/v`.
-{{< highlight txt >}}
-git tag -s storage/vx.x.x -m "release vx.x.x"
-git push origin storage/vx.x.x
-{{< / highlight >}}
-5. Wait for CI and check that the GitHub release was published.
-
-Congratulations, you just released the storage submodule!
diff --git a/docs/extensions/storage/spaces.md b/docs/extensions/storage/spaces.md
index bb24cf98e2..45b445c79c 100644
--- a/docs/extensions/storage/spaces.md
+++ b/docs/extensions/storage/spaces.md
@@ -47,16 +47,16 @@ This is an extract of an element of the list spaces response. An entire object h
Having introduced the above, one can refer to a Drive with the following URL format:
```console
-'https://localhost:9200/graph/v1.0/Drive(1284d238-aa92-42ce-bdc4-0b0000009157!07c26b3a-9944-4f2b-ab33-b0b326fc7570")
+'https://localhost:9200/graph/v1.0/drives/1284d238-aa92-42ce-bdc4-0b0000009157!07c26b3a-9944-4f2b-ab33-b0b326fc7570
```
Updating an entity attribute:
```console
-curl -X PATCH 'https://localhost:9200/graph/v1.0/Drive("1284d238-aa92-42ce-bdc4-0b0000009157!07c26b3a-9944-4f2b-ab33-b0b326fc7570)' -d '{"name":"42"}' -v
+curl -X PATCH 'https://localhost:9200/graph/v1.0/drives/1284d238-aa92-42ce-bdc4-0b0000009157!07c26b3a-9944-4f2b-ab33-b0b326fc7570' -d '{"name":"42"}' -v
```
-The previous URL resource path segment (`Drive(1284d238-aa92-42ce-bdc4-0b0000009157!07c26b3a-9944-4f2b-ab33-b0b326fc7570)`) is parsed and handed over to the storage registry in order to apply the patch changes in the body, in this case update the space name attribute to `42`. Since space names are not unique we only support addressing them by their unique identifiers, any other query would render too ambiguous and explode in complexity.
+The previous URL resource path segment (`1284d238-aa92-42ce-bdc4-0b0000009157!07c26b3a-9944-4f2b-ab33-b0b326fc7570`) is parsed and handed over to the storage registry in order to apply the patch changes in the body, in this case update the space name attribute to `42`. Since space names are not unique we only support addressing them by their unique identifiers, any other query would render too ambiguous and explode in complexity.
### Updating a space description
@@ -140,13 +140,13 @@ Upload a 6 bytes file:
Query the quota again:
```json
-...
-"quota": {
+{
+ "quota": {
"remaining": 4,
"total": 10,
"used": 6
-},
-...
+ }
+}
```
Now attempt to upload 5 bytes to the space:
@@ -176,3 +176,33 @@ The request will fail with `507 Insufficient Storage`:
##### Considerations
- If a Space quota is updated to unlimited, the upper limit is the entire available space on disk
+{{< hint warning >}}
+
+The current implementation in oCIS might not yet fully reflect this concept. Feel free to add links to ADRs, PRs and Issues in short warning boxes like this.
+
+{{< /hint >}}
+
+## Storage Spaces
+A storage *space* is a logical concept. It organizes a set of [*resources*]({{< ref "#resources" >}}) in a hierarchical tree. It has a single *owner* (*user* or *group*),
+a *quota*, *permissions* and is identified by a `storage space id`.
+
+{{< svg src="extensions/storage/static/storagespace.drawio.svg" >}}
+
+Examples would be every user's personal storage *space*, project storage *spaces* or group storage *spaces*. While they all serve different purposes and may or may not have workflows like anti virus scanning enabled, we need a way to identify and manage these subtrees in a generic way. By creating a dedicated concept for them this becomes easier and literally makes the codebase cleaner. A storage [*Spaces Registry*]({{< ref "./spacesregistry.md" >}}) then allows listing the capabilities of storage *spaces*, e.g. free space, quota, owner, syncable, root etag, upload workflow steps, ...
+
+Finally, a logical `storage space id` is not tied to a specific [*spaces provider*]({{< ref "./spacesprovider.md" >}}). If the [*storage driver*]({{< ref "./storagedrivers.md" >}}) supports it, we can import existing files including their `file id`, which makes it possible to move storage *spaces* between [*spaces providers*]({{< ref "./spacesprovider.md" >}}) to implement storage classes, e.g. with or without archival, workflows, on SSDs or HDDs.
+
+## Shares
+*To be clarified: we are aware that [*storage spaces*]({{< ref "#storage-spaces" >}}) may be too 'heavywheight' for ad hoc sharing with groups. That being said, there is no technical reason why group shares should not be treated like storage [*spaces*]({{< ref "#storage-spaces" >}}) that users can provision themselves. They would share the quota with the users home or personal storage [*space*]({{< ref "#storage-spaces" >}}) and the share initiator would be the sole owner. Technically, the mechanism of treating a share like a new storage [*space*]({{< ref "#storage-spaces" >}}) would be the same. This obviously also extends to user shares and even file individual shares that would be wrapped in a virtual collection. It would also become possible to share collections of arbitrary files in a single storage space, e.g. the ten best pictures from a large album.*
+
+## Notes
+
+We can implement [ListStorageSpaces](https://cs3org.github.io/cs3apis/#cs3.storage.provider.v1beta1.ListStorageSpacesRequest) by either
+- iterating over the root of the storage and treating every folder following the `` as a `home` *storage space*,
+- iterating over the root of the storage and treating every folder following a new `` as a `project` *storage space*, or
+- iterating over the root of the storage and treating every folder following a generic `` as a *storage space* for a configurable space type, or
+- we allow configuring a map of `space type` to `layout` (based on the [CreateStorageSpaceRequest](https://cs3org.github.io/cs3apis/#cs3.storage.provider.v1beta1.CreateStorageSpaceRequest)) which would allow things like
+```
+home=/var/lib/ocis/storage/home/{{substr 0 1 .Owner.Username}}/{{.Owner.Username}}
+spaces=/spaces/var/lib/ocis/storage/projects/{{.Name}}
+```
diff --git a/docs/extensions/storage/architecture.md b/docs/extensions/storage/spacesprovider.md
similarity index 76%
rename from docs/extensions/storage/architecture.md
rename to docs/extensions/storage/spacesprovider.md
index 3e1a6cf67d..5a706ff514 100644
--- a/docs/extensions/storage/architecture.md
+++ b/docs/extensions/storage/spacesprovider.md
@@ -1,12 +1,25 @@
---
-title: "Architecture"
+title: "Spaces Provider"
date: 2018-05-02T00:00:00+00:00
-weight: 10
+weight: 6
geekdocRepo: https://github.com/owncloud/ocis
geekdocEditPath: edit/master/docs/extensions/storage
-geekdocFilePath: architecture.md
+geekdocFilePath: spacesprovider.md
---
+{{< hint warning >}}
+
+The current implementation in oCIS might not yet fully reflect this concept. Feel free to add links to ADRs, PRs and Issues in short warning boxes like this.
+
+{{< /hint >}}
+
+## Spaces Provider
+A *storage provider* manages [*resources*]({{< ref "#resources" >}}) identified by a [*reference*]({{< ref "#references" >}})
+by accessing a [*storage system*]({{< ref "#storage-systems" >}}) with a [*storage driver*]({{< ref "./storagedrivers.md" >}}).
+
+{{< svg src="extensions/storage/static/spacesprovider.drawio.svg" >}}
+
+
## Frontend
The oCIS frontend service starts all services that handle incoming HTTP requests:
@@ -36,18 +49,19 @@ The ocdav service not only handles all WebDAV requests under `(remote.php/)(web)
| *Note: existing folder sync pairs in legacy clients will break when moving the user home down in the path hierarchy* |||||
| `(remote.php/)webdav/home` | ocdav | storageprovider | `/home` | | |
| `(remote.php/)webdav/users` | ocdav | storageprovider | `/users` | | |
-| `(remote.php/)dav/files/` | ocdav | storageprovider | `/users/` | | |
+| `(remote.php/)dav/files/` | ocdav | storageprovider | `/users/` | | |
| *Spaces concept also needs a new endpoint:* |||||
-| `(remote.php/)dav/spaces//` | ocdav | storageregistry & storageprovider | bypass path based namespace and directly talk to the responsible storage provider using a relative path | [spaces concept](https://github.com/owncloud/ocis/pull/1827) needs to point to [*storage spaces*]({{< ref "./terminology.md#storage-spaces" >}}) or a global endpoint | allow accessing spaces, listing is done by the graph api |
+| `(remote.php/)dav/spaces//` | ocdav | storageregistry & storageprovider | bypass path based namespace and directly talk to the responsible storage provider using a relative path | [spaces concept](https://github.com/owncloud/ocis/pull/1827) needs to point to storage [*spaces*]({{< ref "./spaces.md" >}}) | allow accessing spaces, listing is done by the graph api |
-The correct endpoint for a users home [*storage space*]({{< ref "./terminology.md#storage-spaces" >}}) in oc10 is `remote.php/dav/files/`. In oc10 All requests at this endpoint use a path based reference that is relative to the users home. In oCIS this can be configured and defaults to `/home` as well. Other API endpoints like ocs and the web UI still expect this to be the users home.
+The correct endpoint for a users home storage [*space*]({{< ref "./spaces.md" >}}) in oc10 is `remote.php/dav/files/`. In oc10 all requests at this endpoint use a path based reference that is relative to the users home. In oCIS this can be configured and defaults to `/home` as well. Other API endpoints like ocs and the web UI still expect this to be the users home.
In oc10 we originally had `remote.php/webdav` which would render the current users home [*storage space*]({{< ref "./terminology.md#storage-spaces" >}}). The early versions (pre OC7) would jail all received shares into a `remote.php/webdav/shares` subfolder. The semantics for syncing such a folder are [not trivially predictable](https://github.com/owncloud/core/issues/5349), which is why we made shares [freely mountable](https://github.com/owncloud/core/pull/8026) anywhere in the users home.
The current reva implementation jails shares into a `remote.php/webdav/Shares` folder for performance reasons. Obviously, this brings back the [special semantics for syncing](https://github.com/owncloud/product/issues/7). In the future we will follow [a different solution](https://github.com/owncloud/product/issues/302) and jail the received shares into a dedicated `/shares` space, on the same level as `/home` and `/spaces`. We will add a dedicated [API to list all *storage spaces*](https://github.com/owncloud/ocis/pull/1827) a user has access to and where they are mounted in the users *namespace*.
{{< hint warning >}}
+TODO rewrite this hint with `/dav/spaces`
Existing folder sync pairs in legacy clients will break when moving the user home down in the path hierarchy like CernBox did.
For legacy clients the `remote.php/webdav` endpoint will no longer list the users home directly, but instead present the different types of storage spaces:
- `remote.php/webdav/home`: the users home is pushed down into a new `home` [*storage space*]({{< ref "./terminology.md#storage-spaces" >}})
@@ -55,11 +69,6 @@ For legacy clients the `remote.php/webdav` endpoint will no longer list the user
- `remote.php/webdav/spaces`: other [*storage spaces*]({{< ref "./terminology.md#storage-spaces" >}}) the user has access to, e.g. group or project drives
{{< /hint >}}
-{{< hint warning >}}
-An alternative would be to introduce a new `remote.php/dav/spaces` or `remote.php/dav/global` endpoint. However, `remote.php/dav` properly follows the WebDAV RFCs strictly. To ensure that all resources under that [*namespace*]({{< ref "./terminology.md#namespaces" >}}) are scoped to the user the URL would have to include the principal like `remote.php/dav/spaces/`, a precondition for e.g. WebDAV [RFC5397](https://tools.ietf.org/html/rfc5397). For a history lesson start at [Replace WebDAV with REST
-owncloud/core#12504](https://github.com/owncloud/core/issues/12504#issuecomment-65218491) which spawned [Add extra layer in DAV to accomodate for other services like versions, trashbin, etc owncloud/core#12543](https://github.com/owncloud/core/issues/12543)
-{{< /hint >}}
-
### Sharing
@@ -92,12 +101,12 @@ The user and public share provider implementations identify the file using the [
The OCM API takes an id based reference on the CS3 api, even if the OCM HTTP endpoint takes a path argument. *@jfd: Why? Does it not need the owner? It only stores the owner of the share, which is always the currently logged in user, when creating a share. Afterwards only the owner can update a share ... so collaborative management of shares is not possible. At least for OCM shares.*
{{< /hint >}}
-### User and Group provisioning
-In oc10 users are identified by a username, which cannot change, because it is used as a foreign key in several tables. For oCIS we are internally identifying users by a UUID, while using the username in the WebDAV and OCS APIs for backwards compatability. To distinguish this in the URLs we are using `` instead of ``. You may have encountered ``, which refers to a template that can be configured to build several path segments by filling in user properties, e.g. the first character of the username (`{{substr 0 1 .Username}}/{{.Username}}`), the identity provider (`{{.Id.Idp}}/{{.Username}}`) or the email (`{{.Mail}}`)
+## REVA Storage Registry
-{{< hint warning >}}
-Make no mistake, the [OCS Provisioning API](https://doc.owncloud.com/server/developer_manual/core/apis/provisioning-api.html) uses `userid` while it actually is the username, because it is what you use to login.
-{{< /hint >}}
+The reva *storage registry* manages the [*CS3 global namespace*]({{< ref "./namespaces.md#cs3-global-namespaces" >}}):
+It is used by the reva *gateway*
+to look up `address` and `port` of the [*storage provider*]({{< ref "#storage-providers" >}})
+that should handle a [*reference*]({{< ref "#references" >}}).
-We are currently working on adding [user management through the CS3 API](https://github.com/owncloud/ocis/pull/1930) to handle user and group provisioning (and deprovisioning).
+{{< svg src="extensions/storage/static/storageregistry.drawio.svg" >}}
\ No newline at end of file
diff --git a/docs/extensions/storage/spacesregistry.md b/docs/extensions/storage/spacesregistry.md
new file mode 100644
index 0000000000..d5be48f8ab
--- /dev/null
+++ b/docs/extensions/storage/spacesregistry.md
@@ -0,0 +1,21 @@
+---
+title: "Spaces Registry"
+date: 2018-05-02T00:00:00+00:00
+weight: 9
+geekdocRepo: https://github.com/owncloud/ocis
+geekdocEditPath: edit/master/docs/extensions/storage
+geekdocFilePath: spacesregistry.md
+---
+
+{{< hint warning >}}
+
+The current implementation in oCIS might not yet fully reflect this concept. Feel free to add links to ADRs, PRs and Issues in short warning boxes like this.
+
+{{< /hint >}}
+
+## Storage Space Registries
+
+A storage *spaces registry* manages the [*namespace*]({{< ref "./namespaces.md" >}}) for a *user*: it is used by *clients* to look up storage spaces a user has access to, the `/dav/spaces` endpoint to access it via WabDAV, and where the client should mount it in the users personal namespace.
+
+{{< svg src="extensions/storage/static/spacesregistry.drawio.svg" >}}
+
diff --git a/docs/extensions/storage/static/namespaces.drawio.svg b/docs/extensions/storage/static/namespaces.drawio.svg
index 5440f46b43..b3baa5895a 100644
--- a/docs/extensions/storage/static/namespaces.drawio.svg
+++ b/docs/extensions/storage/static/namespaces.drawio.svg
@@ -1,4 +1,4 @@
-