From 64a669d69f588a155a94696cac80fb1a45de1736 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Tue, 26 Apr 2022 15:34:16 +0200 Subject: [PATCH 01/15] reenable parallel deployment tests --- .drone.star | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.drone.star b/.drone.star index 188a2c7f3d..3c7cedcf9b 100644 --- a/.drone.star +++ b/.drone.star @@ -110,9 +110,7 @@ config = { "suites": [ "apiShareManagement", ], - # The tests fail after the storage config changes - # They will be fixed later. - "skip": True, + "skip": False, "earlyFail": True, "cron": "nightly", }, @@ -120,9 +118,7 @@ config = { "suites": [ "apiWebdavOperations", ], - # The tests fail after the storage config changes - # They will be fixed later. - "skip": True, + "skip": False, "earlyFail": True, "cron": "nightly", }, From 7cb7daffd011aa156eb32fb74508a659836a6086 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Wed, 4 May 2022 07:57:31 +0200 Subject: [PATCH 02/15] temporalily skip other tests --- .drone.star | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.drone.star b/.drone.star index 3c7cedcf9b..8a9acb8db0 100644 --- a/.drone.star +++ b/.drone.star @@ -82,18 +82,18 @@ config = { "earlyFail": True, }, "localApiTests": { - "skip": False, + "skip": True, "earlyFail": True, }, "apiTests": { "numberOfParts": 10, - "skip": False, + "skip": True, "skipExceptParts": [], "earlyFail": True, }, "uiTests": { "filterTags": "@ocisSmokeTest", - "skip": False, + "skip": True, "skipExceptParts": [], "earlyFail": True, }, From d2de2a775de7ad1d67ba91a92edaa289ececda85 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Wed, 4 May 2022 08:34:36 +0200 Subject: [PATCH 03/15] add OCIS_OIDC_ISSUER config env --- .drone.star | 24 +++--- .../config/ocis/entrypoint-override.sh | 8 ++ .../oc10_ocis_parallel/docker-compose.yml | 83 +++++++++---------- extensions/auth-basic/pkg/config/config.go | 2 +- extensions/auth-bearer/pkg/config/config.go | 2 +- .../graph-explorer/pkg/config/config.go | 2 +- extensions/groups/pkg/config/config.go | 2 +- extensions/idp/pkg/config/config.go | 2 +- extensions/ocs/pkg/config/config.go | 2 +- extensions/proxy/pkg/config/config.go | 2 +- extensions/users/pkg/config/config.go | 2 +- extensions/web/pkg/config/config.go | 2 +- 12 files changed, 67 insertions(+), 66 deletions(-) create mode 100644 deployments/examples/oc10_ocis_parallel/config/ocis/entrypoint-override.sh diff --git a/.drone.star b/.drone.star index 8a9acb8db0..7c19e3a1ad 100644 --- a/.drone.star +++ b/.drone.star @@ -1663,16 +1663,14 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = else: user = "33:33" environment = { + # Keycloak IDP specific configuration + "OCIS_OIDC_ISSUER": "https://keycloak/auth/realms/owncloud", + "WEB_OIDC_METADATA_URL": "https://keycloak/auth/realms/owncloud/.well-known/openid-configuration", + "WEB_OIDC_CLIENT_ID": "ocis-web", + "WEB_OIDC_SCOPE": "openid profile email owncloud", + # external ldap is supposed to be read only "GRAPH_IDENTITY_BACKEND": "cs3", "GRAPH_LDAP_SERVER_WRITE_ENABLED": "false", - # Keycloak IDP specific configuration - "PROXY_OIDC_ISSUER": "https://keycloak/auth/realms/owncloud", - "LDAP_IDP": "https://keycloak/auth/realms/owncloud", - "WEB_OIDC_AUTHORITY": "https://keycloak/auth/realms/owncloud", - "WEB_OIDC_CLIENT_ID": "ocis-web", - "WEB_OIDC_METADATA_URL": "https://keycloak/auth/realms/owncloud/.well-known/openid-configuration", - "AUTH_BEARER_OIDC_ISSUER": "https://keycloak", - "WEB_OIDC_SCOPE": "openid profile email owncloud", # LDAP bind "LDAP_URI": "ldaps://openldap", "LDAP_INSECURE": "true", @@ -1685,19 +1683,19 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = "PROXY_USER_CS3_CLAIM": "userid", # equals STORAGE_LDAP_USER_SCHEMA_UID "LDAP_GROUP_BASE_DN": "ou=testgroups,dc=owncloud,dc=com", "LDAP_GROUP_OBJECTCLASS": "groupOfUniqueNames", - "LDAP_GROUPFILTER": "(objectclass=owncloud)", "LDAP_GROUP_SCHEMA_DISPLAYNAME": "cn", "LDAP_GROUP_SCHEMA_ID": "cn", "LDAP_GROUP_SCHEMA_MAIL": "mail", "LDAP_GROUP_SCHEMA_MEMBER": "cn", + "LDAP_GROUPFILTER": "(objectclass=owncloud)", + "LDAP_LOGIN_ATTRIBUTES": "uid,mail", "LDAP_USER_BASE_DN": "ou=testusers,dc=owncloud,dc=com", "LDAP_USER_OBJECTCLASS": "posixAccount", - "LDAP_USERFILTER": "(objectclass=owncloud)", - "LDAP_USER_SCHEMA_USERNAME": "cn", "LDAP_USER_SCHEMA_DISPLAYNAME": "displayname", - "LDAP_USER_SCHEMA_MAIL": "mail", "LDAP_USER_SCHEMA_ID": "ownclouduuid", - "LDAP_LOGIN_ATTRIBUTES": "uid,mail", + "LDAP_USER_SCHEMA_MAIL": "mail", + "LDAP_USER_SCHEMA_USERNAME": "cn", + "LDAP_USERFILTER": "(objectclass=owncloud)", # ownCloudSQL storage driver "STORAGE_USERS_DRIVER": "owncloudsql", "STORAGE_USERS_OWNCLOUDSQL_DATADIR": "/mnt/data/files", diff --git a/deployments/examples/oc10_ocis_parallel/config/ocis/entrypoint-override.sh b/deployments/examples/oc10_ocis_parallel/config/ocis/entrypoint-override.sh new file mode 100644 index 0000000000..dcbef6736a --- /dev/null +++ b/deployments/examples/oc10_ocis_parallel/config/ocis/entrypoint-override.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -e + +ocis init || true # will only initialize once + +#chmod 744 -R /etc/ocis +#setpriv --reuid=33 --regid=33 --clear-groups +ocis server diff --git a/deployments/examples/oc10_ocis_parallel/docker-compose.yml b/deployments/examples/oc10_ocis_parallel/docker-compose.yml index 56886b6d2f..053eeebfc5 100644 --- a/deployments/examples/oc10_ocis_parallel/docker-compose.yml +++ b/deployments/examples/oc10_ocis_parallel/docker-compose.yml @@ -50,60 +50,55 @@ services: user: "33:33" # equals the user "www-data" for oC10 environment: # Keycloak IDP specific configuration - PROXY_OIDC_ISSUER: https://${KEYCLOAK_DOMAIN:-keycloak.owncloud.test}/auth/realms/${KEYCLOAK_REALM:-owncloud} - WEB_OIDC_AUTHORITY: https://${KEYCLOAK_DOMAIN:-keycloak.owncloud.test}/auth/realms/${KEYCLOAK_REALM:-owncloud} - WEB_OIDC_CLIENT_ID: ocis-web + OCIS_OIDC_ISSUER: https://${KEYCLOAK_DOMAIN:-keycloak.owncloud.test}/auth/realms/${KEYCLOAK_REALM:-owncloud} WEB_OIDC_METADATA_URL: https://${KEYCLOAK_DOMAIN:-keycloak.owncloud.test}/auth/realms/${KEYCLOAK_REALM:-owncloud}/.well-known/openid-configuration - STORAGE_OIDC_ISSUER: https://${KEYCLOAK_DOMAIN:-keycloak.owncloud.test} - STORAGE_LDAP_IDP: https://${KEYCLOAK_DOMAIN:-keycloak.owncloud.test}/auth/realms/${KEYCLOAK_REALM:-owncloud} + WEB_OIDC_CLIENT_ID: ocis-web WEB_OIDC_SCOPE: openid profile email owncloud + # external ldap is supposed to be read only + GRAPH_IDENTITY_BACKEND: cs3 + GRAPH_LDAP_SERVER_WRITE_ENABLED: "false" # LDAP bind - STORAGE_LDAP_URI: "ldaps://openldap" - STORAGE_LDAP_INSECURE: "true" - STORAGE_LDAP_BIND_DN: "cn=admin,dc=owncloud,dc=com" - STORAGE_LDAP_BIND_PASSWORD: ${LDAP_ADMIN_PASSWORD:-admin} + LDAP_URI: "ldaps://openldap" + LDAP_INSECURE: "true" + LDAP_BIND_DN: "cn=admin,dc=owncloud,dc=com" + LDAP_BIND_PASSWORD: ${LDAP_ADMIN_PASSWORD:-admin} # LDAP user settings PROXY_AUTOPROVISION_ACCOUNTS: "true" # automatically create users when they login PROXY_ACCOUNT_BACKEND_TYPE: cs3 # proxy should get users from CS3APIS (which gets it from LDAP) PROXY_USER_OIDC_CLAIM: ocis.user.uuid # claim was added in Keycloak PROXY_USER_CS3_CLAIM: userid # equals STORAGE_LDAP_USER_SCHEMA_UID - STORAGE_LDAP_GROUP_BASE_DN: "dc=owncloud,dc=com" - STORAGE_LDAP_GROUP_SCHEMA_DISPLAYNAME: "cn" - STORAGE_LDAP_GROUP_SCHEMA_GID_NUMBER: "gidnumber" - STORAGE_LDAP_GROUP_SCHEMA_ID: "cn" - STORAGE_LDAP_GROUP_SCHEMA_MAIL: "mail" - STORAGE_LDAP_GROUP_SCHEMA_MEMBER: "cn" - STORAGE_LDAP_GROUP_OBJECTCLASS: "groupOfUniqueNames" - STORAGE_LDAP_GROUPFILTER: "(objectclass=owncloud)" - STORAGE_LDAP_USER_BASE_DN: "dc=owncloud,dc=com" - STORAGE_LDAP_USER_SCHEMA_USERNAME: "cn" - STORAGE_LDAP_USER_SCHEMA_DISPLAYNAME: "displayname" - STORAGE_LDAP_USER_SCHEMA_GID_NUMBER: "gidnumber" - STORAGE_LDAP_USER_SCHEMA_MAIL: "mail" - STORAGE_LDAP_USER_SCHEMA_UID_NUMBER: "uidnumber" - STORAGE_LDAP_USER_SCHEMA_ID: "ownclouduuid" - STORAGE_LDAP_LOGIN_ATTRIBUTES: "uid,mail" + LDAP_GROUP_BASE_DN: "dc=owncloud,dc=com" + LDAP_GROUP_OBJECTCLASS: "groupOfUniqueNames" + LDAP_GROUP_SCHEMA_DISPLAYNAME: "cn" + LDAP_GROUP_SCHEMA_ID: "cn" + LDAP_GROUP_SCHEMA_MAIL: "mail" + LDAP_GROUP_SCHEMA_MEMBER: "cn" + LDAP_GROUPFILTER: "(objectclass=owncloud)" + LDAP_LOGIN_ATTRIBUTES: "uid,mail" + LDAP_USER_BASE_DN: "dc=owncloud,dc=com" + LDAP_USER_OBJECTCLASS: "posixAccount" + LDAP_USER_SCHEMA_DISPLAYNAME: "displayname" + LDAP_USER_SCHEMA_ID: "ownclouduuid" + LDAP_USER_SCHEMA_MAIL: "mail" + LDAP_USER_SCHEMA_USERNAME: "cn" + LDAP_USERFILTER: "(objectclass=owncloud)" # ownCloudSQL storage driver - STORAGE_USERS_DRIVER: owncloudsql - STORAGE_SYSTEM_DRIVER: ocis # keep system data on ocis storage since this are only small files atm - STORAGE_USERS_DRIVER_OWNCLOUDSQL_DATADIR: /mnt/data/files - STORAGE_USERS_DRIVER_OWNCLOUDSQL_UPLOADINFO_DIR: /tmp - STORAGE_USERS_DRIVER_OWNCLOUDSQL_SHARE_FOLDER: "/Shares" - STORAGE_USERS_DRIVER_OWNCLOUDSQL_LAYOUT: "{{.Username}}" - STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBUSERNAME: owncloud - STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBPASSWORD: owncloud - STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBHOST: oc10-db - STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBPORT: 3306 - STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBNAME: owncloud - STORAGE_USERS_DRIVER_OWNCLOUDSQL_REDIS_ADDR: redis:6379 # TODO: redis is not yet supported + STORAGE_USERS_DRIVER: "owncloudsql" + STORAGE_USERS_OWNCLOUDSQL_DATADIR: "/mnt/data/files" + STORAGE_USERS_OWNCLOUDSQL_SHARE_FOLDER: "/Shares" + STORAGE_USERS_OWNCLOUDSQL_LAYOUT: "{{.Username}}" + STORAGE_USERS_OWNCLOUDSQL_DB_USERNAME: "owncloud" + STORAGE_USERS_OWNCLOUDSQL_DB_PASSWORD: "owncloud" + STORAGE_USERS_OWNCLOUDSQL_DB_HOST: "oc10-db" + STORAGE_USERS_OWNCLOUDSQL_DB_PORT: 3306 + STORAGE_USERS_OWNCLOUDSQL_DB_NAME: "owncloud" # ownCloudSQL sharing driver - STORAGE_SHARING_USER_DRIVER: owncloudsql - STORAGE_SHARING_USER_SQL_USERNAME: owncloud - STORAGE_SHARING_USER_SQL_PASSWORD: owncloud - STORAGE_SHARING_USER_SQL_HOST: oc10-db - STORAGE_SHARING_USER_SQL_PORT: 3306 - STORAGE_SHARING_USER_SQL_NAME: owncloud - + SHARING_USER_DRIVER: "owncloudsql" + SHARING_USER_OWNCLOUDSQL_DB_USERNAME: "owncloud" + SHARING_USER_OWNCLOUDSQL_DB_PASSWORD: "owncloud" + SHARING_USER_OWNCLOUDSQL_DB_HOST: "oc10-db" + SHARING_USER_OWNCLOUDSQL_DB_PORT: 330 + SHARING_USER_OWNCLOUDSQL_DB_NAME: "owncloud" # ownCloud storage readonly OCIS_STORAGE_READ_ONLY: "false" # TODO: conflict with OWNCLOUDSQL -> https://github.com/owncloud/ocis/issues/2303 # General oCIS config diff --git a/extensions/auth-basic/pkg/config/config.go b/extensions/auth-basic/pkg/config/config.go index 69b5fba146..ef4392f843 100644 --- a/extensions/auth-basic/pkg/config/config.go +++ b/extensions/auth-basic/pkg/config/config.go @@ -81,7 +81,7 @@ type LDAPProvider struct { UserObjectClass string `yaml:"user_object_filter" env:"LDAP_USER_OBJECTCLASS;AUTH_BASIC_LDAP_USER_OBJECTCLASS"` GroupObjectClass string `yaml:"group_object_class" env:"LDAP_GROUP_OBJECTCLASS;AUTH_BASIC_LDAP_GROUP_OBJECTCLASS"` LoginAttributes []string `yaml:"login_attributes" env:"LDAP_LOGIN_ATTRIBUTES;AUTH_BASIC_LDAP_LOGIN_ATTRIBUTES"` - IDP string `yaml:"idp" env:"OCIS_URL;AUTH_BASIC_IDP_URL"` + IDP string `yaml:"idp" env:"OCIS_URL;OCIS_OIDC_ISSUER;AUTH_BASIC_IDP_URL"` UserSchema LDAPUserSchema `yaml:"user_schema"` GroupSchema LDAPGroupSchema `yaml:"group_schema"` } diff --git a/extensions/auth-bearer/pkg/config/config.go b/extensions/auth-bearer/pkg/config/config.go index 7c1390e2af..962fb06d88 100644 --- a/extensions/auth-bearer/pkg/config/config.go +++ b/extensions/auth-bearer/pkg/config/config.go @@ -57,7 +57,7 @@ type GRPCConfig struct { } type OIDC struct { - Issuer string `yaml:"issuer" env:"OCIS_URL;AUTH_BEARER_OIDC_ISSUER"` + Issuer string `yaml:"issuer" env:"OCIS_URL;OCIS_OIDC_ISSUER;AUTH_BEARER_OIDC_ISSUER"` Insecure bool `yaml:"insecure" env:"OCIS_INSECURE;AUTH_BEARER_OIDC_INSECURE"` IDClaim string `yaml:"id_claim" env:"AUTH_BEARER_OIDC_ID_CLAIM"` UIDClaim string `yaml:"uid_claim" env:"AUTH_BEARER_OIDC_UID_CLAIM"` diff --git a/extensions/graph-explorer/pkg/config/config.go b/extensions/graph-explorer/pkg/config/config.go index 4301472d4a..afe2230057 100644 --- a/extensions/graph-explorer/pkg/config/config.go +++ b/extensions/graph-explorer/pkg/config/config.go @@ -26,7 +26,7 @@ type Config struct { // GraphExplorer defines the available graph-explorer configuration. type GraphExplorer struct { ClientID string `yaml:"client_id" env:"GRAPH_EXPLORER_CLIENT_ID"` - Issuer string `yaml:"issuer" env:"OCIS_URL;GRAPH_EXPLORER_ISSUER"` + Issuer string `yaml:"issuer" env:"OCIS_URL;OCIS_OIDC_ISSUER;GRAPH_EXPLORER_ISSUER"` GraphURLBase string `yaml:"graph_url_base" env:"OCIS_URL;GRAPH_EXPLORER_GRAPH_URL_BASE"` GraphURLPath string `yaml:"graph_url_path" env:"GRAPH_EXPLORER_GRAPH_URL_PATH"` } diff --git a/extensions/groups/pkg/config/config.go b/extensions/groups/pkg/config/config.go index e0ff9c1ce5..ac7859ef3c 100644 --- a/extensions/groups/pkg/config/config.go +++ b/extensions/groups/pkg/config/config.go @@ -80,7 +80,7 @@ type LDAPDriver struct { UserObjectClass string `yaml:"user_object_class" env:"LDAP_USER_OBJECTCLASS;GROUPS_LDAP_USER_OBJECTCLASS"` GroupObjectClass string `yaml:"group_object_class" env:"LDAP_GROUP_OBJECTCLASS;GROUPS_LDAP_GROUP_OBJECTCLASS"` LoginAttributes []string `yaml:"login_attributes" env:"LDAP_LOGIN_ATTRIBUTES;GROUPS_LDAP_LOGIN_ATTRIBUTES"` - IDP string `yaml:"idp" env:"OCIS_URL;GROUPS_IDP_URL"` + IDP string `yaml:"idp" env:"OCIS_URL;OCIS_OIDC_ISSUER;GROUPS_IDP_URL"` UserSchema LDAPUserSchema `yaml:"user_schema"` GroupSchema LDAPGroupSchema `yaml:"group_schema"` } diff --git a/extensions/idp/pkg/config/config.go b/extensions/idp/pkg/config/config.go index ab6a90c1a4..b239e1be40 100644 --- a/extensions/idp/pkg/config/config.go +++ b/extensions/idp/pkg/config/config.go @@ -55,7 +55,7 @@ type Settings struct { // don't change the order of elements in this struct // it needs to match github.com/libregraph/lico/bootstrap.Settings - Iss string `yaml:"iss" env:"OCIS_URL;IDP_ISS"` + Iss string `yaml:"iss" env:"OCIS_URL;OCIS_OIDC_ISSUER;IDP_ISS"` IdentityManager string `yaml:"identity_manager" env:"IDP_IDENTITY_MANAGER"` diff --git a/extensions/ocs/pkg/config/config.go b/extensions/ocs/pkg/config/config.go index 53ba58276b..a105abf455 100644 --- a/extensions/ocs/pkg/config/config.go +++ b/extensions/ocs/pkg/config/config.go @@ -34,5 +34,5 @@ type Config struct { // is based in the combination of IDP hostname + UserID. For more information see: // https://github.com/cs3org/reva/blob/4fd0229f13fae5bc9684556a82dbbd0eced65ef9/pkg/storage/utils/decomposedfs/node/node.go#L856-L865 type IdentityManagement struct { - Address string `yaml:"address" env:"OCIS_URL;OCS_IDM_ADDRESS"` + Address string `yaml:"address" env:"OCIS_URL;OCIS_OIDC_ISSUER;OCS_IDM_ADDRESS"` } diff --git a/extensions/proxy/pkg/config/config.go b/extensions/proxy/pkg/config/config.go index a8c394f36c..f71e000667 100644 --- a/extensions/proxy/pkg/config/config.go +++ b/extensions/proxy/pkg/config/config.go @@ -83,7 +83,7 @@ type AuthMiddleware struct { // OIDC is the config for the OpenID-Connect middleware. If set the proxy will try to authenticate every request // with the configured oidc-provider type OIDC struct { - Issuer string `yaml:"issuer" env:"OCIS_URL;PROXY_OIDC_ISSUER"` + Issuer string `yaml:"issuer" env:"OCIS_URL;OCIS_OIDC_ISSUER;PROXY_OIDC_ISSUER"` Insecure bool `yaml:"insecure" env:"OCIS_INSECURE;PROXY_OIDC_INSECURE"` UserinfoCache UserinfoCache `yaml:"user_info_cache"` } diff --git a/extensions/users/pkg/config/config.go b/extensions/users/pkg/config/config.go index 81f27896ee..57b3f95c1d 100644 --- a/extensions/users/pkg/config/config.go +++ b/extensions/users/pkg/config/config.go @@ -84,7 +84,7 @@ type LDAPDriver struct { UserObjectClass string `yaml:"user_object_class" env:"LDAP_USER_OBJECTCLASS;USERS_LDAP_USER_OBJECTCLASS"` GroupObjectClass string `yaml:"group_object_class" env:"LDAP_GROUP_OBJECTCLASS;USERS_LDAP_GROUP_OBJECTCLASS"` LoginAttributes []string `yaml:"login_attributes" env:"LDAP_LOGIN_ATTRIBUTES;USERS_LDAP_LOGIN_ATTRIBUTES"` - IDP string `yaml:"idp" env:"OCIS_URL;USERS_IDP_URL"` + IDP string `yaml:"idp" env:"OCIS_URL;OCIS_OIDC_ISSUER;USERS_IDP_URL"` UserSchema LDAPUserSchema `yaml:"user_schema"` GroupSchema LDAPGroupSchema `yaml:"group_schema"` } diff --git a/extensions/web/pkg/config/config.go b/extensions/web/pkg/config/config.go index a01b8f82b1..e289109b1b 100644 --- a/extensions/web/pkg/config/config.go +++ b/extensions/web/pkg/config/config.go @@ -44,7 +44,7 @@ type WebConfig struct { // OIDC defines the available oidc configuration type OIDC struct { MetadataURL string `json:"metadata_url,omitempty" yaml:"metadata_url" env:"WEB_OIDC_METADATA_URL"` - Authority string `json:"authority,omitempty" yaml:"authority" env:"OCIS_URL;WEB_OIDC_AUTHORITY"` + Authority string `json:"authority,omitempty" yaml:"authority" env:"OCIS_URL;OCIS_OIDC_ISSUER;WEB_OIDC_AUTHORITY"` ClientID string `json:"client_id,omitempty" yaml:"client_id" env:"WEB_OIDC_CLIENT_ID"` ResponseType string `json:"response_type,omitempty" yaml:"response_type" env:"WEB_OIDC_RESPONSE_TYPE"` Scope string `json:"scope,omitempty" yaml:"scope" env:"WEB_OIDC_SCOPE"` From 0e2c85d6358ffa2a440c56355bff9cda4bf9f865 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Wed, 4 May 2022 08:50:36 +0200 Subject: [PATCH 04/15] fix nats command naming --- ocis/pkg/command/natsserver.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ocis/pkg/command/natsserver.go b/ocis/pkg/command/natsserver.go index 17fa6023ed..b021172064 100644 --- a/ocis/pkg/command/natsserver.go +++ b/ocis/pkg/command/natsserver.go @@ -10,11 +10,11 @@ import ( "github.com/urfave/cli/v2" ) -// NatsServerCommand is the entrypoint for the nats server command. -func NatsServerCommand(cfg *config.Config) *cli.Command { +// NatsCommand is the entrypoint for the Nats command. +func NatsCommand(cfg *config.Config) *cli.Command { return &cli.Command{ - Name: "nats-server", - Usage: "start nats server", + Name: cfg.Nats.Service.Name, + Usage: subcommandDescription(cfg.Nats.Service.Name), Category: "extensions", Before: func(c *cli.Context) error { if err := parser.ParseConfig(cfg); err != nil { @@ -29,5 +29,5 @@ func NatsServerCommand(cfg *config.Config) *cli.Command { } func init() { - register.AddCommand(NatsServerCommand) + register.AddCommand(NatsCommand) } From 0cfc36562945674c1e03fbc6fe0b24871e5669d8 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Thu, 5 May 2022 17:03:46 +0200 Subject: [PATCH 05/15] update proxy config --- .../drone/ocis/proxy.yaml | 156 ++++++++++++++---- 1 file changed, 126 insertions(+), 30 deletions(-) diff --git a/tests/parallelDeployAcceptance/drone/ocis/proxy.yaml b/tests/parallelDeployAcceptance/drone/ocis/proxy.yaml index 50f89d6ac3..0c678461e8 100644 --- a/tests/parallelDeployAcceptance/drone/ocis/proxy.yaml +++ b/tests/parallelDeployAcceptance/drone/ocis/proxy.yaml @@ -6,48 +6,144 @@ policy_selector: policies: - name: ocis routes: - - endpoint: "/" + - type: "" + method: "" + endpoint: / backend: http://localhost:9100 - - endpoint: "/.well-known/" + service: "" + apache_vhost: false + - type: "" + method: "" + endpoint: /.well-known/ backend: http://localhost:9130 + service: "" + apache_vhost: false + - type: "" + method: "" + endpoint: /konnect/ + backend: http://localhost:9130 + service: "" + apache_vhost: false + - type: "" + method: "" + endpoint: /signin/ + backend: http://localhost:9130 + service: "" + apache_vhost: false + - type: "" + method: "" + endpoint: /archiver + backend: http://localhost:9140 + service: "" + apache_vhost: false - type: regex - endpoint: "/ocs/v[12].php/cloud/user/signing-key" + method: "" + endpoint: /ocs/v[12].php/cloud/user/signing-key backend: http://localhost:9110 - - endpoint: "/ocs/" + service: "" + apache_vhost: false + - type: "" + method: "" + endpoint: /ocs/ backend: http://localhost:9140 + service: "" + apache_vhost: false - type: query - endpoint: "/remote.php/?preview=1" + method: "" + endpoint: /remote.php/?preview=1 backend: http://localhost:9115 - - endpoint: "/remote.php/" - service: "ocdav" - - endpoint: "/dav/" - service: "ocdav" - - endpoint: "/webdav/" - service: "ocdav" - - endpoint: "/status.php" - service: "ocdav" - - endpoint: "/index.php/" - service: "ocdav" - - endpoint: "/apps/" - service: "ocdav" - - endpoint: "/index.php/login" - backend: http://localhost:9100 - - endpoint: "/login" - backend: http://localhost:9100 - - endpoint: "/data" + service: "" + apache_vhost: false + - type: "" + method: REPORT + endpoint: /remote.php/dav/ + backend: http://localhost:9115 + service: "" + apache_vhost: false + - type: "" + method: "" + endpoint: /remote.php/ + backend: "" + service: ocdav + apache_vhost: false + - type: "" + method: "" + endpoint: /dav/ + backend: "" + service: ocdav + apache_vhost: false + - type: "" + method: "" + endpoint: /webdav/ + backend: "" + service: ocdav + apache_vhost: false + - type: "" + method: "" + endpoint: /status.php + backend: "" + service: ocdav + apache_vhost: false + - type: "" + method: "" + endpoint: /index.php/ + backend: "" + service: ocdav + apache_vhost: false + - type: "" + method: "" + endpoint: /apps/ + backend: "" + service: ocdav + apache_vhost: false + - type: "" + method: "" + endpoint: /data backend: http://localhost:9140 - - endpoint: "/graph/" + service: "" + apache_vhost: false + - type: "" + method: "" + endpoint: /app/ + backend: http://localhost:9140 + service: "" + apache_vhost: false + - type: "" + method: "" + endpoint: /graph/ backend: http://localhost:9120 - - endpoint: "/app/" - backend: http://localhost:9140 - - endpoint: "/archiver" - backend: http://localhost:9140 - - endpoint: "/graph-explorer/" + service: "" + apache_vhost: false + - type: "" + method: "" + endpoint: /graph-explorer backend: http://localhost:9135 - - endpoint: "/api/v0/settings" + service: "" + apache_vhost: false + - type: "" + method: "" + endpoint: /api/v0/accounts + backend: http://localhost:9181 + service: "" + apache_vhost: false + - type: "" + method: "" + endpoint: /accounts.js + backend: http://localhost:9181 + service: "" + apache_vhost: false + - type: "" + method: "" + endpoint: /api/v0/settings backend: http://localhost:9190 - - endpoint: "/settings.js" + service: "" + apache_vhost: false + - type: "" + method: "" + endpoint: /settings.js backend: http://localhost:9190 + service: "" + apache_vhost: false - name: oc10 routes: - endpoint: "/" From 4a9d0f297fd844202e8176a43f81fa7ebf006667 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Fri, 6 May 2022 09:27:39 +0200 Subject: [PATCH 06/15] update proxy conf --- .../oc10_ocis_parallel/config/ocis/proxy.yaml | 152 ++++++++++++++---- .../oc10_ocis_parallel/docker-compose.yml | 25 ++- 2 files changed, 148 insertions(+), 29 deletions(-) mode change 100644 => 100755 deployments/examples/oc10_ocis_parallel/config/ocis/proxy.yaml diff --git a/deployments/examples/oc10_ocis_parallel/config/ocis/proxy.yaml b/deployments/examples/oc10_ocis_parallel/config/ocis/proxy.yaml old mode 100644 new mode 100755 index a472adcf12..0c678461e8 --- a/deployments/examples/oc10_ocis_parallel/config/ocis/proxy.yaml +++ b/deployments/examples/oc10_ocis_parallel/config/ocis/proxy.yaml @@ -6,46 +6,144 @@ policy_selector: policies: - name: ocis routes: - - endpoint: "/" + - type: "" + method: "" + endpoint: / backend: http://localhost:9100 - - endpoint: "/.well-known/" + service: "" + apache_vhost: false + - type: "" + method: "" + endpoint: /.well-known/ backend: http://localhost:9130 + service: "" + apache_vhost: false + - type: "" + method: "" + endpoint: /konnect/ + backend: http://localhost:9130 + service: "" + apache_vhost: false + - type: "" + method: "" + endpoint: /signin/ + backend: http://localhost:9130 + service: "" + apache_vhost: false + - type: "" + method: "" + endpoint: /archiver + backend: http://localhost:9140 + service: "" + apache_vhost: false - type: regex - endpoint: "/ocs/v[12].php/cloud/user/signing-key" + method: "" + endpoint: /ocs/v[12].php/cloud/user/signing-key backend: http://localhost:9110 - - endpoint: "/ocs/" + service: "" + apache_vhost: false + - type: "" + method: "" + endpoint: /ocs/ backend: http://localhost:9140 + service: "" + apache_vhost: false - type: query - endpoint: "/remote.php/?preview=1" + method: "" + endpoint: /remote.php/?preview=1 backend: http://localhost:9115 - - endpoint: "/remote.php/" + service: "" + apache_vhost: false + - type: "" + method: REPORT + endpoint: /remote.php/dav/ + backend: http://localhost:9115 + service: "" + apache_vhost: false + - type: "" + method: "" + endpoint: /remote.php/ + backend: "" + service: ocdav + apache_vhost: false + - type: "" + method: "" + endpoint: /dav/ + backend: "" + service: ocdav + apache_vhost: false + - type: "" + method: "" + endpoint: /webdav/ + backend: "" + service: ocdav + apache_vhost: false + - type: "" + method: "" + endpoint: /status.php + backend: "" + service: ocdav + apache_vhost: false + - type: "" + method: "" + endpoint: /index.php/ + backend: "" + service: ocdav + apache_vhost: false + - type: "" + method: "" + endpoint: /apps/ + backend: "" + service: ocdav + apache_vhost: false + - type: "" + method: "" + endpoint: /data backend: http://localhost:9140 - - endpoint: "/dav/" + service: "" + apache_vhost: false + - type: "" + method: "" + endpoint: /app/ backend: http://localhost:9140 - - endpoint: "/webdav/" - backend: http://localhost:9140 - - endpoint: "/status.php" - backend: http://localhost:9140 - - endpoint: "/index.php/" - backend: http://localhost:9140 - - endpoint: "/index.php/login" - backend: http://localhost:9100 - - endpoint: "/login" - backend: http://localhost:9100 - - endpoint: "/data" - backend: http://localhost:9140 - - endpoint: "/graph/" + service: "" + apache_vhost: false + - type: "" + method: "" + endpoint: /graph/ backend: http://localhost:9120 - - endpoint: "/app/" - backend: http://localhost:9140 - - endpoint: "/archiver" - backend: http://localhost:9140 - - endpoint: "/graph-explorer/" + service: "" + apache_vhost: false + - type: "" + method: "" + endpoint: /graph-explorer backend: http://localhost:9135 - - endpoint: "/api/v0/settings" + service: "" + apache_vhost: false + - type: "" + method: "" + endpoint: /api/v0/accounts + backend: http://localhost:9181 + service: "" + apache_vhost: false + - type: "" + method: "" + endpoint: /accounts.js + backend: http://localhost:9181 + service: "" + apache_vhost: false + - type: "" + method: "" + endpoint: /api/v0/settings backend: http://localhost:9190 - - endpoint: "/settings.js" + service: "" + apache_vhost: false + - type: "" + method: "" + endpoint: /settings.js backend: http://localhost:9190 + service: "" + apache_vhost: false - name: oc10 routes: - endpoint: "/" diff --git a/deployments/examples/oc10_ocis_parallel/docker-compose.yml b/deployments/examples/oc10_ocis_parallel/docker-compose.yml index 053eeebfc5..2cc505ef71 100644 --- a/deployments/examples/oc10_ocis_parallel/docker-compose.yml +++ b/deployments/examples/oc10_ocis_parallel/docker-compose.yml @@ -43,11 +43,30 @@ services: driver: "local" restart: always + ocis-init-volumes: + image: busybox + entrypoint: + - /bin/sh + # prepare the oCIS config volume for oCIS + command: ["-c", "chown -R 33:33 /etc/ocis /var/lib/ocis"] + volumes: + - ocis-config:/etc/ocis + - ocis-data:/var/lib/ocis + ocis: image: owncloud/ocis:${OCIS_DOCKER_TAG:-latest} networks: ocis-net: user: "33:33" # equals the user "www-data" for oC10 + entrypoint: + - /bin/sh + # run ocis init to initialize a configuration file with random secrets + # it will fail on subsequent runs, because the config file already exists + # therefore we ignore the error and then start the ocis server + command: ["-c", "ocis init || true; ocis server"] + #entrypoint: + # - /bin/sh + # - /entrypoint-override.sh environment: # Keycloak IDP specific configuration OCIS_OIDC_ISSUER: https://${KEYCLOAK_DOMAIN:-keycloak.owncloud.test}/auth/realms/${KEYCLOAK_REALM:-owncloud} @@ -97,13 +116,13 @@ services: SHARING_USER_OWNCLOUDSQL_DB_USERNAME: "owncloud" SHARING_USER_OWNCLOUDSQL_DB_PASSWORD: "owncloud" SHARING_USER_OWNCLOUDSQL_DB_HOST: "oc10-db" - SHARING_USER_OWNCLOUDSQL_DB_PORT: 330 + SHARING_USER_OWNCLOUDSQL_DB_PORT: 3306 SHARING_USER_OWNCLOUDSQL_DB_NAME: "owncloud" # ownCloud storage readonly OCIS_STORAGE_READ_ONLY: "false" # TODO: conflict with OWNCLOUDSQL -> https://github.com/owncloud/ocis/issues/2303 # General oCIS config # OCIS_RUN_EXTENSIONS specifies to start all extensions except glauth, idp and accounts. These are replaced by external services - OCIS_RUN_EXTENSIONS: settings,storage-system,graph,graph-explorer,ocs,store,thumbnails,web,webdav,frontend,gateway,users,groups,auth-basic,auth-bearer,auth-machine,storage-users,storage-shares,storage-publiclink,app-provider,sharing,proxy,nats + OCIS_RUN_EXTENSIONS: app-registry,app-provider,audit,auth-basic,auth-bearer,auth-machine,frontend,gateway,graph,graph-explorer,groups,nats,notifications,ocdav,ocs,proxy,search,settings,sharing,storage-system,storage-publiclink,storage-shares,storage-users,store,thumbnails,users,web,webdav OCIS_LOG_LEVEL: ${OCIS_LOG_LEVEL:-error} # make oCIS less verbose OCIS_URL: https://${CLOUD_DOMAIN:-cloud.owncloud.test} PROXY_TLS: "false" # do not use SSL between Traefik and oCIS @@ -120,6 +139,7 @@ services: IDM_CREATE_DEMO_USERS: "${DEMO_USERS:-false}" volumes: - ./config/ocis/proxy.yaml:/etc/ocis/proxy.yaml + - ocis-config:/etc/ocis - ocis-data:/var/lib/ocis # shared volume with oC10 - oc10-data:/mnt/data @@ -334,6 +354,7 @@ services: volumes: certs: + ocis-config: ocis-data: keycloak-postgres-data: oc10-mysql-data: From dfafcb96643bf7399f78641e3226cccaa9eef386 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Fri, 6 May 2022 12:50:38 +0200 Subject: [PATCH 07/15] update proxy config --- .../oc10_ocis_parallel/config/ocis/proxy.yaml | 139 ++++-------------- 1 file changed, 28 insertions(+), 111 deletions(-) diff --git a/deployments/examples/oc10_ocis_parallel/config/ocis/proxy.yaml b/deployments/examples/oc10_ocis_parallel/config/ocis/proxy.yaml index 0c678461e8..49e64f6748 100755 --- a/deployments/examples/oc10_ocis_parallel/config/ocis/proxy.yaml +++ b/deployments/examples/oc10_ocis_parallel/config/ocis/proxy.yaml @@ -6,144 +6,61 @@ policy_selector: policies: - name: ocis routes: - - type: "" - method: "" - endpoint: / + - endpoint: / backend: http://localhost:9100 - service: "" - apache_vhost: false - - type: "" - method: "" - endpoint: /.well-known/ + - endpoint: /.well-known/ backend: http://localhost:9130 - service: "" - apache_vhost: false - - type: "" - method: "" - endpoint: /konnect/ + - endpoint: /konnect/ backend: http://localhost:9130 - service: "" - apache_vhost: false - - type: "" - method: "" - endpoint: /signin/ + - endpoint: /signin/ backend: http://localhost:9130 - service: "" - apache_vhost: false - - type: "" - method: "" - endpoint: /archiver + - endpoint: /archiver backend: http://localhost:9140 - service: "" - apache_vhost: false - type: regex - method: "" - endpoint: /ocs/v[12].php/cloud/user/signing-key + endpoint: /ocs/v[12].php/cloud/(users?|groups) backend: http://localhost:9110 - service: "" - apache_vhost: false - - type: "" - method: "" - endpoint: /ocs/ + - endpoint: /ocs/ backend: http://localhost:9140 - service: "" - apache_vhost: false - type: query - method: "" endpoint: /remote.php/?preview=1 backend: http://localhost:9115 - service: "" - apache_vhost: false - - type: "" - method: REPORT + - method: REPORT endpoint: /remote.php/dav/ backend: http://localhost:9115 - service: "" - apache_vhost: false - - type: "" - method: "" - endpoint: /remote.php/ - backend: "" + - type: query + endpoint: /dav/?preview=1 + backend: http://localhost:9115 + - type: query + endpoint: /webdav/?preview=1 + backend: http://localhost:9115 + - endpoint: /remote.php/ service: ocdav - apache_vhost: false - - type: "" - method: "" - endpoint: /dav/ - backend: "" + - endpoint: /dav/ service: ocdav - apache_vhost: false - - type: "" - method: "" - endpoint: /webdav/ - backend: "" + - endpoint: /webdav/ service: ocdav - apache_vhost: false - - type: "" - method: "" - endpoint: /status.php - backend: "" + - endpoint: /status.php service: ocdav - apache_vhost: false - - type: "" - method: "" - endpoint: /index.php/ - backend: "" + - endpoint: /index.php/ service: ocdav - apache_vhost: false - - type: "" - method: "" - endpoint: /apps/ - backend: "" + - endpoint: /apps/ service: ocdav - apache_vhost: false - - type: "" - method: "" - endpoint: /data + - endpoint: /data backend: http://localhost:9140 - service: "" - apache_vhost: false - - type: "" - method: "" - endpoint: /app/ + - endpoint: /app/ backend: http://localhost:9140 - service: "" - apache_vhost: false - - type: "" - method: "" - endpoint: /graph/ + - endpoint: /graph/ backend: http://localhost:9120 - service: "" - apache_vhost: false - - type: "" - method: "" - endpoint: /graph-explorer + - endpoint: /graph-explorer backend: http://localhost:9135 - service: "" - apache_vhost: false - - type: "" - method: "" - endpoint: /api/v0/accounts + - endpoint: /api/v0/accounts backend: http://localhost:9181 - service: "" - apache_vhost: false - - type: "" - method: "" - endpoint: /accounts.js + - endpoint: /accounts.js backend: http://localhost:9181 - service: "" - apache_vhost: false - - type: "" - method: "" - endpoint: /api/v0/settings + - endpoint: /api/v0/settings backend: http://localhost:9190 - service: "" - apache_vhost: false - - type: "" - method: "" - endpoint: /settings.js + - endpoint: /settings.js backend: http://localhost:9190 - service: "" - apache_vhost: false - name: oc10 routes: - endpoint: "/" From 68be48b95ae00069bb25c43f0b48d7e968ff89d7 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Fri, 6 May 2022 12:52:47 +0200 Subject: [PATCH 08/15] use userprovider in owncloudsql storage and add gateway for sharing driver --- extensions/sharing/pkg/revaconfig/config.go | 1 + extensions/storage-users/pkg/config/config.go | 2 +- .../pkg/config/defaults/defaultconfig.go | 19 ++++++++++--------- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/extensions/sharing/pkg/revaconfig/config.go b/extensions/sharing/pkg/revaconfig/config.go index 6fdb79d3cd..10c2458511 100644 --- a/extensions/sharing/pkg/revaconfig/config.go +++ b/extensions/sharing/pkg/revaconfig/config.go @@ -41,6 +41,7 @@ func SharingConfigFromStruct(cfg *config.Config) map[string]interface{} { "janitor_run_interval": cfg.UserSharingDrivers.SQL.JanitorRunInterval, }, "owncloudsql": map[string]interface{}{ + "gateway_addr": cfg.Reva.Address, "storage_mount_id": cfg.UserSharingDrivers.OwnCloudSQL.UserStorageMountID, "db_username": cfg.UserSharingDrivers.OwnCloudSQL.DBUsername, "db_password": cfg.UserSharingDrivers.OwnCloudSQL.DBPassword, diff --git a/extensions/storage-users/pkg/config/config.go b/extensions/storage-users/pkg/config/config.go index be37a5bfbc..d740644999 100644 --- a/extensions/storage-users/pkg/config/config.go +++ b/extensions/storage-users/pkg/config/config.go @@ -129,7 +129,7 @@ type OwnCloudSQLDriver struct { DBHost string `yaml:"db_host" env:"STORAGE_USERS_OWNCLOUDSQL_DB_HOST"` DBPort int `yaml:"db_port" env:"STORAGE_USERS_OWNCLOUDSQL_DB_PORT"` DBName string `yaml:"db_name" env:"STORAGE_USERS_OWNCLOUDSQL_DB_NAME"` - UsersProviderEndpoint string `yaml:"users_provider_endpoint" env:"STORAGE_USERS_PERMISSION_ENDPOINT;STORAGE_USERS_OWNCLOUDSQL_USERS_PROVIDER_ENDPOINT"` + UsersProviderEndpoint string `yaml:"users_provider_endpoint" env:"STORAGE_USERS_OWNCLOUDSQL_USERS_PROVIDER_ENDPOINT"` } type Events struct { diff --git a/extensions/storage-users/pkg/config/defaults/defaultconfig.go b/extensions/storage-users/pkg/config/defaults/defaultconfig.go index aec7a9d68c..3827096c9c 100644 --- a/extensions/storage-users/pkg/config/defaults/defaultconfig.go +++ b/extensions/storage-users/pkg/config/defaults/defaultconfig.go @@ -45,15 +45,16 @@ func DefaultConfig() *config.Config { Driver: "ocis", Drivers: config.Drivers{ OwnCloudSQL: config.OwnCloudSQLDriver{ - Root: filepath.Join(defaults.BaseDataPath(), "storage", "owncloud"), - ShareFolder: "/Shares", - UserLayout: "{{.Username}}", - UploadInfoDir: filepath.Join(defaults.BaseDataPath(), "storage", "uploadinfo"), - DBUsername: "owncloud", - DBPassword: "owncloud", - DBHost: "", - DBPort: 3306, - DBName: "owncloud", + Root: filepath.Join(defaults.BaseDataPath(), "storage", "owncloud"), + ShareFolder: "/Shares", + UserLayout: "{{.Username}}", + UploadInfoDir: filepath.Join(defaults.BaseDataPath(), "storage", "uploadinfo"), + DBUsername: "owncloud", + DBPassword: "owncloud", + DBHost: "", + DBPort: 3306, + DBName: "owncloud", + UsersProviderEndpoint: "localhost:9144", }, S3NG: config.S3NGDriver{ Root: filepath.Join(defaults.BaseDataPath(), "storage", "users"), From 5e5f350c75e869f866a85cbecea267c2a366141a Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Fri, 6 May 2022 12:54:17 +0200 Subject: [PATCH 09/15] update proxy config for ci --- .../drone/ocis/proxy.yaml | 139 ++++-------------- 1 file changed, 28 insertions(+), 111 deletions(-) diff --git a/tests/parallelDeployAcceptance/drone/ocis/proxy.yaml b/tests/parallelDeployAcceptance/drone/ocis/proxy.yaml index 0c678461e8..49e64f6748 100644 --- a/tests/parallelDeployAcceptance/drone/ocis/proxy.yaml +++ b/tests/parallelDeployAcceptance/drone/ocis/proxy.yaml @@ -6,144 +6,61 @@ policy_selector: policies: - name: ocis routes: - - type: "" - method: "" - endpoint: / + - endpoint: / backend: http://localhost:9100 - service: "" - apache_vhost: false - - type: "" - method: "" - endpoint: /.well-known/ + - endpoint: /.well-known/ backend: http://localhost:9130 - service: "" - apache_vhost: false - - type: "" - method: "" - endpoint: /konnect/ + - endpoint: /konnect/ backend: http://localhost:9130 - service: "" - apache_vhost: false - - type: "" - method: "" - endpoint: /signin/ + - endpoint: /signin/ backend: http://localhost:9130 - service: "" - apache_vhost: false - - type: "" - method: "" - endpoint: /archiver + - endpoint: /archiver backend: http://localhost:9140 - service: "" - apache_vhost: false - type: regex - method: "" - endpoint: /ocs/v[12].php/cloud/user/signing-key + endpoint: /ocs/v[12].php/cloud/(users?|groups) backend: http://localhost:9110 - service: "" - apache_vhost: false - - type: "" - method: "" - endpoint: /ocs/ + - endpoint: /ocs/ backend: http://localhost:9140 - service: "" - apache_vhost: false - type: query - method: "" endpoint: /remote.php/?preview=1 backend: http://localhost:9115 - service: "" - apache_vhost: false - - type: "" - method: REPORT + - method: REPORT endpoint: /remote.php/dav/ backend: http://localhost:9115 - service: "" - apache_vhost: false - - type: "" - method: "" - endpoint: /remote.php/ - backend: "" + - type: query + endpoint: /dav/?preview=1 + backend: http://localhost:9115 + - type: query + endpoint: /webdav/?preview=1 + backend: http://localhost:9115 + - endpoint: /remote.php/ service: ocdav - apache_vhost: false - - type: "" - method: "" - endpoint: /dav/ - backend: "" + - endpoint: /dav/ service: ocdav - apache_vhost: false - - type: "" - method: "" - endpoint: /webdav/ - backend: "" + - endpoint: /webdav/ service: ocdav - apache_vhost: false - - type: "" - method: "" - endpoint: /status.php - backend: "" + - endpoint: /status.php service: ocdav - apache_vhost: false - - type: "" - method: "" - endpoint: /index.php/ - backend: "" + - endpoint: /index.php/ service: ocdav - apache_vhost: false - - type: "" - method: "" - endpoint: /apps/ - backend: "" + - endpoint: /apps/ service: ocdav - apache_vhost: false - - type: "" - method: "" - endpoint: /data + - endpoint: /data backend: http://localhost:9140 - service: "" - apache_vhost: false - - type: "" - method: "" - endpoint: /app/ + - endpoint: /app/ backend: http://localhost:9140 - service: "" - apache_vhost: false - - type: "" - method: "" - endpoint: /graph/ + - endpoint: /graph/ backend: http://localhost:9120 - service: "" - apache_vhost: false - - type: "" - method: "" - endpoint: /graph-explorer + - endpoint: /graph-explorer backend: http://localhost:9135 - service: "" - apache_vhost: false - - type: "" - method: "" - endpoint: /api/v0/accounts + - endpoint: /api/v0/accounts backend: http://localhost:9181 - service: "" - apache_vhost: false - - type: "" - method: "" - endpoint: /accounts.js + - endpoint: /accounts.js backend: http://localhost:9181 - service: "" - apache_vhost: false - - type: "" - method: "" - endpoint: /api/v0/settings + - endpoint: /api/v0/settings backend: http://localhost:9190 - service: "" - apache_vhost: false - - type: "" - method: "" - endpoint: /settings.js + - endpoint: /settings.js backend: http://localhost:9190 - service: "" - apache_vhost: false - name: oc10 routes: - endpoint: "/" From ca2dcf15146e9e5be59fb9145dd8499cd62deeab Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Fri, 6 May 2022 12:55:16 +0200 Subject: [PATCH 10/15] Revert "temporalily skip other tests" This reverts commit 7cb7daffd011aa156eb32fb74508a659836a6086. --- .drone.star | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.drone.star b/.drone.star index 7c19e3a1ad..dc1f4405c5 100644 --- a/.drone.star +++ b/.drone.star @@ -82,18 +82,18 @@ config = { "earlyFail": True, }, "localApiTests": { - "skip": True, + "skip": False, "earlyFail": True, }, "apiTests": { "numberOfParts": 10, - "skip": True, + "skip": False, "skipExceptParts": [], "earlyFail": True, }, "uiTests": { "filterTags": "@ocisSmokeTest", - "skip": True, + "skip": False, "skipExceptParts": [], "earlyFail": True, }, From 6b5447ef7b517dc8b3361aef7855207d995f7b46 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Fri, 6 May 2022 12:57:54 +0200 Subject: [PATCH 11/15] remove unused entrypoint script --- .../oc10_ocis_parallel/config/ocis/entrypoint-override.sh | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 deployments/examples/oc10_ocis_parallel/config/ocis/entrypoint-override.sh diff --git a/deployments/examples/oc10_ocis_parallel/config/ocis/entrypoint-override.sh b/deployments/examples/oc10_ocis_parallel/config/ocis/entrypoint-override.sh deleted file mode 100644 index dcbef6736a..0000000000 --- a/deployments/examples/oc10_ocis_parallel/config/ocis/entrypoint-override.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -set -e - -ocis init || true # will only initialize once - -#chmod 744 -R /etc/ocis -#setpriv --reuid=33 --regid=33 --clear-groups -ocis server From 90937e027c013228e7e1a9326f6be5995904af7d Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Fri, 6 May 2022 14:27:39 +0200 Subject: [PATCH 12/15] update extensions to run --- .drone.star | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.star b/.drone.star index dc1f4405c5..92713ef074 100644 --- a/.drone.star +++ b/.drone.star @@ -1715,7 +1715,7 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = "SHARING_USER_OWNCLOUDSQL_DB_NAME": "owncloud", # General oCIS config # OCIS_RUN_EXTENSIONS specifies to start all extensions except glauth, idp and accounts. These are replaced by external services - "OCIS_RUN_EXTENSIONS": "settings,storage-system,graph,graph-explorer,ocs,store,thumbnails,web,webdav,frontend,gateway,users,groups,auth-basic,auth-bearer,auth-machine,storage-users,storage-shares,storage-publiclink,app-provider,sharing,proxy,nats,ocdav", + "OCIS_RUN_EXTENSIONS": "app-registry,app-provider,audit,auth-basic,auth-bearer,auth-machine,frontend,gateway,graph,graph-explorer,groups,nats,notifications,ocdav,ocs,proxy,search,settings,sharing,storage-system,storage-publiclink,storage-shares,storage-users,store,thumbnails,users,web,webdav", "OCIS_LOG_LEVEL": "info", "OCIS_URL": OCIS_URL, "OCIS_BASE_DATA_PATH": "/mnt/data/ocis", From 5c1ffb5038b51a15aa2316095ea9c390b10ab58c Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Sat, 7 May 2022 12:59:29 +0200 Subject: [PATCH 13/15] adapt proxy config and remove uneeded config --- .drone.star | 14 +++++--------- .../oc10_ocis_parallel/config/ocis/proxy.yaml | 12 ++++++------ .../oc10_ocis_parallel/docker-compose.yml | 17 ++++------------- .../drone/ocis/proxy.yaml | 12 ++++++------ 4 files changed, 21 insertions(+), 34 deletions(-) diff --git a/.drone.star b/.drone.star index 92713ef074..8607097140 100644 --- a/.drone.star +++ b/.drone.star @@ -1637,7 +1637,7 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = if not testing_parallel_deploy: user = "0:0" environment = { - "OCIS_URL": "https://ocis-server:9200", + "OCIS_URL": OCIS_URL, "GATEWAY_GRPC_ADDR": "0.0.0.0:9142", # cs3api-validator needs the cs3api gatway exposed "STORAGE_USERS_DRIVER": "%s" % (storage), "STORAGE_USERS_DRIVER_LOCAL_ROOT": "/srv/app/tmp/ocis/local/root", @@ -1669,7 +1669,7 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = "WEB_OIDC_CLIENT_ID": "ocis-web", "WEB_OIDC_SCOPE": "openid profile email owncloud", # external ldap is supposed to be read only - "GRAPH_IDENTITY_BACKEND": "cs3", + "GRAPH_IDENTITY_BACKEND": "ldap", "GRAPH_LDAP_SERVER_WRITE_ENABLED": "false", # LDAP bind "LDAP_URI": "ldaps://openldap", @@ -1677,19 +1677,17 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = "LDAP_BIND_DN": "cn=admin,dc=owncloud,dc=com", "LDAP_BIND_PASSWORD": "admin", # LDAP user settings - "PROXY_AUTOPROVISION_ACCOUNTS": "true", # automatically create users when they login - "PROXY_ACCOUNT_BACKEND_TYPE": "cs3", # proxy should get users from CS3APIS (which gets it from LDAP) "PROXY_USER_OIDC_CLAIM": "ocis.user.uuid", # claim was added in Keycloak "PROXY_USER_CS3_CLAIM": "userid", # equals STORAGE_LDAP_USER_SCHEMA_UID - "LDAP_GROUP_BASE_DN": "ou=testgroups,dc=owncloud,dc=com", + "LDAP_GROUP_BASE_DN": "ou=TestGroups,dc=owncloud,dc=com", "LDAP_GROUP_OBJECTCLASS": "groupOfUniqueNames", "LDAP_GROUP_SCHEMA_DISPLAYNAME": "cn", "LDAP_GROUP_SCHEMA_ID": "cn", "LDAP_GROUP_SCHEMA_MAIL": "mail", "LDAP_GROUP_SCHEMA_MEMBER": "cn", "LDAP_GROUPFILTER": "(objectclass=owncloud)", - "LDAP_LOGIN_ATTRIBUTES": "uid,mail", - "LDAP_USER_BASE_DN": "ou=testusers,dc=owncloud,dc=com", + "LDAP_LOGIN_ATTRIBUTES": "uid", + "LDAP_USER_BASE_DN": "ou=TestUsers,dc=owncloud,dc=com", "LDAP_USER_OBJECTCLASS": "posixAccount", "LDAP_USER_SCHEMA_DISPLAYNAME": "displayname", "LDAP_USER_SCHEMA_ID": "ownclouduuid", @@ -1721,8 +1719,6 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = "OCIS_BASE_DATA_PATH": "/mnt/data/ocis", "OCIS_CONFIG_DIR": "/etc/ocis", "PROXY_ENABLE_BASIC_AUTH": "true", - "IDM_CREATE_DEMO_USERS": True, - "IDM_ADMIN_PASSWORD": "admin", # override the random admin password from `ocis init` } wait_for_ocis = { "name": "wait-for-ocis-server", diff --git a/deployments/examples/oc10_ocis_parallel/config/ocis/proxy.yaml b/deployments/examples/oc10_ocis_parallel/config/ocis/proxy.yaml index 49e64f6748..7c9a1c21b5 100755 --- a/deployments/examples/oc10_ocis_parallel/config/ocis/proxy.yaml +++ b/deployments/examples/oc10_ocis_parallel/config/ocis/proxy.yaml @@ -34,17 +34,17 @@ policies: endpoint: /webdav/?preview=1 backend: http://localhost:9115 - endpoint: /remote.php/ - service: ocdav + service: com.owncloud.web.ocdav - endpoint: /dav/ - service: ocdav + service: com.owncloud.web.ocdav - endpoint: /webdav/ - service: ocdav + service: com.owncloud.web.ocdav - endpoint: /status.php - service: ocdav + service: com.owncloud.web.ocdav - endpoint: /index.php/ - service: ocdav + service: com.owncloud.web.ocdav - endpoint: /apps/ - service: ocdav + service: com.owncloud.web.ocdav - endpoint: /data backend: http://localhost:9140 - endpoint: /app/ diff --git a/deployments/examples/oc10_ocis_parallel/docker-compose.yml b/deployments/examples/oc10_ocis_parallel/docker-compose.yml index 2cc505ef71..7a74976a49 100644 --- a/deployments/examples/oc10_ocis_parallel/docker-compose.yml +++ b/deployments/examples/oc10_ocis_parallel/docker-compose.yml @@ -74,7 +74,7 @@ services: WEB_OIDC_CLIENT_ID: ocis-web WEB_OIDC_SCOPE: openid profile email owncloud # external ldap is supposed to be read only - GRAPH_IDENTITY_BACKEND: cs3 + GRAPH_IDENTITY_BACKEND: ldap GRAPH_LDAP_SERVER_WRITE_ENABLED: "false" # LDAP bind LDAP_URI: "ldaps://openldap" @@ -82,19 +82,17 @@ services: LDAP_BIND_DN: "cn=admin,dc=owncloud,dc=com" LDAP_BIND_PASSWORD: ${LDAP_ADMIN_PASSWORD:-admin} # LDAP user settings - PROXY_AUTOPROVISION_ACCOUNTS: "true" # automatically create users when they login - PROXY_ACCOUNT_BACKEND_TYPE: cs3 # proxy should get users from CS3APIS (which gets it from LDAP) PROXY_USER_OIDC_CLAIM: ocis.user.uuid # claim was added in Keycloak PROXY_USER_CS3_CLAIM: userid # equals STORAGE_LDAP_USER_SCHEMA_UID - LDAP_GROUP_BASE_DN: "dc=owncloud,dc=com" + LDAP_GROUP_BASE_DN: "ou=groups,dc=owncloud,dc=com" LDAP_GROUP_OBJECTCLASS: "groupOfUniqueNames" LDAP_GROUP_SCHEMA_DISPLAYNAME: "cn" LDAP_GROUP_SCHEMA_ID: "cn" LDAP_GROUP_SCHEMA_MAIL: "mail" LDAP_GROUP_SCHEMA_MEMBER: "cn" LDAP_GROUPFILTER: "(objectclass=owncloud)" - LDAP_LOGIN_ATTRIBUTES: "uid,mail" - LDAP_USER_BASE_DN: "dc=owncloud,dc=com" + LDAP_LOGIN_ATTRIBUTES: "uid" + LDAP_USER_BASE_DN: "ou=users,dc=owncloud,dc=com" LDAP_USER_OBJECTCLASS: "posixAccount" LDAP_USER_SCHEMA_DISPLAYNAME: "displayname" LDAP_USER_SCHEMA_ID: "ownclouduuid" @@ -126,17 +124,10 @@ services: OCIS_LOG_LEVEL: ${OCIS_LOG_LEVEL:-error} # make oCIS less verbose OCIS_URL: https://${CLOUD_DOMAIN:-cloud.owncloud.test} PROXY_TLS: "false" # do not use SSL between Traefik and oCIS - # change default secrets - OCIS_JWT_SECRET: ${OCIS_JWT_SECRET:-Pive-Fumkiu4} - STORAGE_TRANSFER_SECRET: ${STORAGE_TRANSFER_SECRET:-replace-me-with-a-transfer-secret} - OCIS_MACHINE_AUTH_API_KEY: ${OCIS_MACHINE_AUTH_API_KEY:-change-me-please} # INSECURE: needed if oCIS / Traefik is using self generated certificates OCIS_INSECURE: "${INSECURE:-false}" # basic auth (not recommended, but needed for eg. WebDav clients that do not support OpenID Connect) PROXY_ENABLE_BASIC_AUTH: "${PROXY_ENABLE_BASIC_AUTH:-false}" - # demo users - ACCOUNTS_DEMO_USERS_AND_GROUPS: "${DEMO_USERS:-false}" # deprecated, remove after switching to LibreIDM - IDM_CREATE_DEMO_USERS: "${DEMO_USERS:-false}" volumes: - ./config/ocis/proxy.yaml:/etc/ocis/proxy.yaml - ocis-config:/etc/ocis diff --git a/tests/parallelDeployAcceptance/drone/ocis/proxy.yaml b/tests/parallelDeployAcceptance/drone/ocis/proxy.yaml index 49e64f6748..7c9a1c21b5 100644 --- a/tests/parallelDeployAcceptance/drone/ocis/proxy.yaml +++ b/tests/parallelDeployAcceptance/drone/ocis/proxy.yaml @@ -34,17 +34,17 @@ policies: endpoint: /webdav/?preview=1 backend: http://localhost:9115 - endpoint: /remote.php/ - service: ocdav + service: com.owncloud.web.ocdav - endpoint: /dav/ - service: ocdav + service: com.owncloud.web.ocdav - endpoint: /webdav/ - service: ocdav + service: com.owncloud.web.ocdav - endpoint: /status.php - service: ocdav + service: com.owncloud.web.ocdav - endpoint: /index.php/ - service: ocdav + service: com.owncloud.web.ocdav - endpoint: /apps/ - service: ocdav + service: com.owncloud.web.ocdav - endpoint: /data backend: http://localhost:9140 - endpoint: /app/ From 17a97f48cd128a863228f721c84715182612ef8e Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Mon, 9 May 2022 11:21:27 +0200 Subject: [PATCH 14/15] bring back uid,mail as login attributes --- .drone.star | 2 +- deployments/examples/oc10_ocis_parallel/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.star b/.drone.star index 8607097140..bed3a13d4b 100644 --- a/.drone.star +++ b/.drone.star @@ -1686,7 +1686,7 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = "LDAP_GROUP_SCHEMA_MAIL": "mail", "LDAP_GROUP_SCHEMA_MEMBER": "cn", "LDAP_GROUPFILTER": "(objectclass=owncloud)", - "LDAP_LOGIN_ATTRIBUTES": "uid", + "LDAP_LOGIN_ATTRIBUTES": "uid,mail", "LDAP_USER_BASE_DN": "ou=TestUsers,dc=owncloud,dc=com", "LDAP_USER_OBJECTCLASS": "posixAccount", "LDAP_USER_SCHEMA_DISPLAYNAME": "displayname", diff --git a/deployments/examples/oc10_ocis_parallel/docker-compose.yml b/deployments/examples/oc10_ocis_parallel/docker-compose.yml index 7a74976a49..f08ecba822 100644 --- a/deployments/examples/oc10_ocis_parallel/docker-compose.yml +++ b/deployments/examples/oc10_ocis_parallel/docker-compose.yml @@ -91,7 +91,7 @@ services: LDAP_GROUP_SCHEMA_MAIL: "mail" LDAP_GROUP_SCHEMA_MEMBER: "cn" LDAP_GROUPFILTER: "(objectclass=owncloud)" - LDAP_LOGIN_ATTRIBUTES: "uid" + LDAP_LOGIN_ATTRIBUTES: "uid,mail" LDAP_USER_BASE_DN: "ou=users,dc=owncloud,dc=com" LDAP_USER_OBJECTCLASS: "posixAccount" LDAP_USER_SCHEMA_DISPLAYNAME: "displayname" From 10c956121a86ed6bac5401e0a35d9ac5b2086916 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Mon, 9 May 2022 11:35:25 +0200 Subject: [PATCH 15/15] Revert "bring back uid,mail as login attributes" This reverts commit 17a97f48cd128a863228f721c84715182612ef8e. --- .drone.star | 2 +- deployments/examples/oc10_ocis_parallel/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.star b/.drone.star index bed3a13d4b..8607097140 100644 --- a/.drone.star +++ b/.drone.star @@ -1686,7 +1686,7 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = "LDAP_GROUP_SCHEMA_MAIL": "mail", "LDAP_GROUP_SCHEMA_MEMBER": "cn", "LDAP_GROUPFILTER": "(objectclass=owncloud)", - "LDAP_LOGIN_ATTRIBUTES": "uid,mail", + "LDAP_LOGIN_ATTRIBUTES": "uid", "LDAP_USER_BASE_DN": "ou=TestUsers,dc=owncloud,dc=com", "LDAP_USER_OBJECTCLASS": "posixAccount", "LDAP_USER_SCHEMA_DISPLAYNAME": "displayname", diff --git a/deployments/examples/oc10_ocis_parallel/docker-compose.yml b/deployments/examples/oc10_ocis_parallel/docker-compose.yml index f08ecba822..7a74976a49 100644 --- a/deployments/examples/oc10_ocis_parallel/docker-compose.yml +++ b/deployments/examples/oc10_ocis_parallel/docker-compose.yml @@ -91,7 +91,7 @@ services: LDAP_GROUP_SCHEMA_MAIL: "mail" LDAP_GROUP_SCHEMA_MEMBER: "cn" LDAP_GROUPFILTER: "(objectclass=owncloud)" - LDAP_LOGIN_ATTRIBUTES: "uid,mail" + LDAP_LOGIN_ATTRIBUTES: "uid" LDAP_USER_BASE_DN: "ou=users,dc=owncloud,dc=com" LDAP_USER_OBJECTCLASS: "posixAccount" LDAP_USER_SCHEMA_DISPLAYNAME: "displayname"