Merge branch 'master' into try-gookikt-config

This commit is contained in:
A.Unger
2021-11-11 13:04:38 +01:00
39 changed files with 1139 additions and 793 deletions
+2
View File
@@ -453,6 +453,7 @@ def localApiTests(ctx, storage, suite, accounts_hash_difficulty = 4):
"BEHAT_SUITE": suite,
"BEHAT_FILTER_TAGS": "~@skip&&~@skipOnOcis-%s-Storage" % ("OC" if storage == "owncloud" else "OCIS"),
"PATH_TO_CORE": "/srv/app/testrunner",
"EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-localAPI-on-%s-storage.md" % (storage.upper()),
"UPLOAD_DELETE_WAIT_TIME": "1" if storage == "owncloud" else 0,
},
"commands": [
@@ -1466,6 +1467,7 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = []):
"IDP_IDENTIFIER_REGISTRATION_CONF": "/drone/src/tests/config/drone/identifier-registration.yml",
"OCIS_LOG_LEVEL": "error",
"SETTINGS_DATA_PATH": "/srv/app/tmp/ocis/settings",
"OCIS_INSECURE": "true",
}
# Pass in "default" accounts_hash_difficulty to not set this environment variable.
+8 -2
View File
@@ -7,12 +7,18 @@
"request": "launch",
"mode": "debug",
"program": "${workspaceFolder}/ocis/cmd/ocis",
"args": ["server"],
"args": [
"server"
],
"env": {
// log settings for human developers
"OCIS_LOG_LEVEL": "debug",
"OCIS_LOG_PRETTY": "true",
"OCIS_LOG_COLOR": "true",
"PROXY_ENABLE_BASIC_AUTH": "true"
// enable basic auth for dev setup so that we can use curl for testing
"PROXY_ENABLE_BASIC_AUTH": "true",
// set insecure options because we don't have valid certificates in dev environments
"OCIS_INSECURE": "true",
}
}
]
+23
View File
@@ -12,6 +12,7 @@ The following sections list the changes for unreleased.
* Bugfix - Fix opening images in media viewer for some usernames: [#2738](https://github.com/owncloud/ocis/pull/2738)
* Bugfix - Fix error logging when there is no thumbnail for a file: [#2702](https://github.com/owncloud/ocis/pull/2702)
* Bugfix - Don't announce resharing via capabilities: [#2690](https://github.com/owncloud/ocis/pull/2690)
* Change - Make all insecure options configurable and change the default to false: [#2700](https://github.com/owncloud/ocis/issues/2700)
* Enhancement - Add API to list all spaces: [#2692](https://github.com/owncloud/ocis/pull/2692)
* Enhancement - Update reva to v1.16: [#2737](https://github.com/owncloud/ocis/pull/2737)
@@ -62,6 +63,28 @@ The following sections list the changes for unreleased.
https://github.com/owncloud/ocis/pull/2690
* Change - Make all insecure options configurable and change the default to false: [#2700](https://github.com/owncloud/ocis/issues/2700)
We had several hard-coded 'insecure' flags. These options are now configurable and default to
false. Also we changed all other 'insecure' flags with a previous default of true to false.
In development environments using self signed certs (the default) you now need to set these
flags:
``` PROXY_OIDC_INSECURE=true STORAGE_FRONTEND_APPPROVIDER_INSECURE=true
STORAGE_FRONTEND_ARCHIVER_INSECURE=true STORAGE_FRONTEND_OCDAV_INSECURE=true
STORAGE_HOME_DATAPROVIDER_INSECURE=true
STORAGE_METADATA_DATAPROVIDER_INSECURE=true STORAGE_OIDC_INSECURE=true
STORAGE_USERS_DATAPROVIDER_INSECURE=true THUMBNAILS_CS3SOURCE_INSECURE=true
THUMBNAILS_WEBDAVSOURCE_INSECURE=true ```
As an alternative you also can set a single flag, which configures all options together:
``` OCIS_INSECURE=true ```
https://github.com/owncloud/ocis/issues/2700
https://github.com/owncloud/ocis/pull/2745
* Enhancement - Add API to list all spaces: [#2692](https://github.com/owncloud/ocis/pull/2692)
Added a graph endpoint to enable users with the `list-all-spaces` permission to list all
+2 -1
View File
@@ -35,6 +35,7 @@ OCIS_MODULES = \
# bin file definitions
PHP_CS_FIXER=php -d zend.enable_gc=0 vendor-bin/owncloud-codestyle/vendor/bin/php-cs-fixer
PHP_CODESNIFFER=vendor-bin/php_codesniffer/vendor/bin/phpcs
PHP_CODEBEAUTIFIER=vendor-bin/php_codesniffer/vendor/bin/phpcbf
PHAN=php -d zend.enable_gc=0 vendor-bin/phan/vendor/bin/phan
PHPSTAN=php -d zend.enable_gc=0 vendor-bin/phpstan/vendor/bin/phpstan
@@ -213,7 +214,7 @@ test-php-style: vendor-bin/owncloud-codestyle/vendor vendor-bin/php_codesniffer/
.PHONY: test-php-style-fix
test-php-style-fix: vendor-bin/owncloud-codestyle/vendor
$(PHP_CS_FIXER) fix -v --diff --allow-risky yes
$(PHP_CODEBEAUTIFIER) --cache --runtime-set ignore_warnings_on_exit --standard=phpcs.xml tests/acceptance
vendor-bin/owncloud-codestyle/vendor: vendor/bamarni/composer-bin-plugin vendor-bin/owncloud-codestyle/composer.lock
composer bin owncloud-codestyle install --no-progress
+27
View File
@@ -0,0 +1,27 @@
Change: Make all insecure options configurable and change the default to false
We had several hard-coded 'insecure' flags. These options are now configurable and default to false. Also we changed all other 'insecure' flags with a previous default of true to false.
In development environments using self signed certs (the default) you now need to set these flags:
```
PROXY_OIDC_INSECURE=true
STORAGE_FRONTEND_APPPROVIDER_INSECURE=true
STORAGE_FRONTEND_ARCHIVER_INSECURE=true
STORAGE_FRONTEND_OCDAV_INSECURE=true
STORAGE_HOME_DATAPROVIDER_INSECURE=true
STORAGE_METADATA_DATAPROVIDER_INSECURE=true
STORAGE_OIDC_INSECURE=true
STORAGE_USERS_DATAPROVIDER_INSECURE=true
THUMBNAILS_CS3SOURCE_INSECURE=true
THUMBNAILS_WEBDAVSOURCE_INSECURE=true
```
As an alternative you also can set a single flag, which configures all options together:
```
OCIS_INSECURE=true
```
https://github.com/owncloud/ocis/issues/2700
https://github.com/owncloud/ocis/pull/2745
@@ -79,12 +79,13 @@ services:
OCIS_URL: https://${OCIS_DOMAIN:-ocis.owncloud.test}
OCIS_DOMAIN: ${OCIS_DOMAIN:-ocis.owncloud.test}
OCIS_LOG_LEVEL: ${OCIS_LOG_LEVEL:-error} # make oCIS less verbose
PROXY_OIDC_INSECURE: "${INSECURE:-false}" # needed if Traefik is using self generated certificates
PROXY_TLS: "false" # do not use SSL between Traefik and oCIS
# change default secrets
OCIS_JWT_SECRET: ${OCIS_JWT_SECRET:-Pive-Fumkiu4}
STORAGE_TRANSFER_SECRET: ${STORAGE_TRANSFER_SECRET:-replace-me-with-a-transfer-secret}
OCIS_MACHINE_AUTH_API_KEY: ${OCIS_MACHINE_AUTH_API_KEY:-change-me-please}
# INSECURE: needed if oCIS / Traefik is using self generated certificates
OCIS_INSECURE: "${INSECURE:-false}"
volumes:
- ./config/ocis/entrypoint-override.sh:/entrypoint-override.sh
- ./config/ocis/web-config.dist.json:/config/web-config.dist.json
@@ -109,13 +109,14 @@ services:
# General oCIS config
OCIS_LOG_LEVEL: ${OCIS_LOG_LEVEL:-error} # make oCIS less verbose
OCIS_URL: https://${CLOUD_DOMAIN:-cloud.owncloud.test}
PROXY_OIDC_INSECURE: "${INSECURE:-false}" # needed if Traefik is using self generated certificates
PROXY_TLS: "false" # do not use SSL between Traefik and oCIS
PROXY_CONFIG_FILE: "/var/tmp/ocis/.config/proxy-config.json"
# change default secrets
OCIS_JWT_SECRET: ${OCIS_JWT_SECRET:-Pive-Fumkiu4}
STORAGE_TRANSFER_SECRET: ${STORAGE_TRANSFER_SECRET:-replace-me-with-a-transfer-secret}
OCIS_MACHINE_AUTH_API_KEY: ${OCIS_MACHINE_AUTH_API_KEY:-change-me-please}
# INSECURE: needed if oCIS / Traefik is using self generated certificates
OCIS_INSECURE: "${INSECURE:-false}"
volumes:
- ./config/ocis/entrypoint-override.sh:/entrypoint-override.sh
- ./config/ocis/proxy.json:/etc/ocis/proxy.json
@@ -53,7 +53,6 @@ services:
OCIS_URL: https://${OCIS_DOMAIN:-ocis.owncloud.test}
OCIS_DOMAIN: ${OCIS_DOMAIN:-ocis.owncloud.test}
OCIS_LOG_LEVEL: ${OCIS_LOG_LEVEL:-error} # make oCIS less verbose
PROXY_OIDC_INSECURE: "${INSECURE:-false}" # needed if Traefik is using self generated certificates
PROXY_TLS: "false" # do not use SSL between Traefik and oCIS
# change default secrets
IDP_LDAP_BIND_PASSWORD: ${IDP_LDAP_BIND_PASSWORD:-idp}
@@ -65,6 +64,8 @@ services:
WEB_UI_CONFIG: "/var/tmp/ocis/.config/web-config.json"
# make settings service available to oCIS Hello
SETTINGS_GRPC_ADDR: 0.0.0.0:9191
# INSECURE: needed if oCIS / Traefik is using self generated certificates
OCIS_INSECURE: "${INSECURE:-false}"
volumes:
- ./config/ocis/entrypoint-override.sh:/entrypoint-override.sh
- ./config/ocis/web-config.dist.json:/config/web-config.dist.json
@@ -62,7 +62,6 @@ services:
# general config
OCIS_URL: https://${OCIS_DOMAIN:-ocis.owncloud.test}
OCIS_LOG_LEVEL: ${OCIS_LOG_LEVEL:-error} # make oCIS less verbose
PROXY_OIDC_INSECURE: "${INSECURE:-false}" # needed if Traefik is using self generated certificates
PROXY_TLS: "false" # do not use SSL between Traefik and oCIS
ACCOUNTS_DEMO_USERS_AND_GROUPS: false # don't generate demo users
# change default secrets
@@ -71,6 +70,8 @@ services:
OCIS_JWT_SECRET: ${OCIS_JWT_SECRET:-Pive-Fumkiu4}
STORAGE_TRANSFER_SECRET: ${STORAGE_TRANSFER_SECRET:-replace-me-with-a-transfer-secret}
OCIS_MACHINE_AUTH_API_KEY: ${OCIS_MACHINE_AUTH_API_KEY:-change-me-please}
# INSECURE: needed if oCIS / Traefik is using self generated certificates
OCIS_INSECURE: "${INSECURE:-false}"
volumes:
- ./config/ocis/entrypoint-override.sh:/entrypoint-override.sh
- ocis-data:/var/lib/ocis
@@ -52,7 +52,6 @@ services:
environment:
OCIS_URL: https://${OCIS_DOMAIN:-ocis.owncloud.test}
OCIS_LOG_LEVEL: ${OCIS_LOG_LEVEL:-error} # make oCIS less verbose
PROXY_OIDC_INSECURE: "${INSECURE:-false}" # needed if Traefik is using self generated certificates
PROXY_TLS: "false" # do not use SSL between Traefik and oCIS
# change default secrets
IDP_LDAP_BIND_PASSWORD: ${IDP_LDAP_BIND_PASSWORD:-idp}
@@ -70,6 +69,8 @@ services:
STORAGE_USERS_DRIVER_S3NG_ACCESS_KEY: ${MINIO_ACCESS_KEY:-ocis}
STORAGE_USERS_DRIVER_S3NG_SECRET_KEY: ${MINIO_SECRET_KEY:-ocis-secret-key}
STORAGE_USERS_DRIVER_S3NG_BUCKET: ${MINIO_BUCKET:-ocis-bucket}
# INSECURE: needed if oCIS / Traefik is using self generated certificates
OCIS_INSECURE: "${INSECURE:-false}"
volumes:
- ./config/ocis/entrypoint-override.sh:/entrypoint-override.sh
- ocis-data:/var/lib/ocis
@@ -52,7 +52,6 @@ services:
environment:
OCIS_URL: https://${OCIS_DOMAIN:-ocis.owncloud.test}
OCIS_LOG_LEVEL: ${OCIS_LOG_LEVEL:-error} # make oCIS less verbose
PROXY_OIDC_INSECURE: "${INSECURE:-false}" # needed if Traefik is using self generated certificates
PROXY_TLS: "false" # do not use SSL between Traefik and oCIS
# change default secrets
IDP_LDAP_BIND_PASSWORD: ${IDP_LDAP_BIND_PASSWORD:-idp}
@@ -60,6 +59,8 @@ services:
OCIS_JWT_SECRET: ${OCIS_JWT_SECRET:-Pive-Fumkiu4}
STORAGE_TRANSFER_SECRET: ${STORAGE_TRANSFER_SECRET:-replace-me-with-a-transfer-secret}
OCIS_MACHINE_AUTH_API_KEY: ${OCIS_MACHINE_AUTH_API_KEY:-change-me-please}
# INSECURE: needed if oCIS / Traefik is using self generated certificates
OCIS_INSECURE: "${INSECURE:-false}"
volumes:
- ./config/ocis/entrypoint-override.sh:/entrypoint-override.sh
- ocis-data:/var/lib/ocis
@@ -58,7 +58,6 @@ services:
OCIS_URL: https://${OCIS_DOMAIN:-ocis.owncloud.test}
OCIS_DOMAIN: ${OCIS_DOMAIN:-ocis.owncloud.test}
OCIS_LOG_LEVEL: ${OCIS_LOG_LEVEL:-error} # make oCIS less verbose
PROXY_OIDC_INSECURE: "${INSECURE:-false}" # needed if Traefik is using self generated certificates
PROXY_TLS: "false" # do not use SSL between Traefik and oCIS
# change default secrets
IDP_LDAP_BIND_PASSWORD: ${IDP_LDAP_BIND_PASSWORD:-idp}
@@ -69,6 +68,8 @@ services:
# app registry
STORAGE_GATEWAY_GRPC_ADDR: 0.0.0.0:9142 # make the REVA gateway accessible to the app drivers
STORAGE_APP_REGISTRY_MIMETYPES_JSON: /var/tmp/ocis/app-config/mimetypes.json
# INSECURE: needed if oCIS / Traefik is using self generated certificates
OCIS_INSECURE: "${INSECURE:-false}"
volumes:
- ./config/ocis/entrypoint-override.sh:/entrypoint-override.sh
- ./config/ocis/mimetypes.json:/var/tmp/ocis/app-config/mimetypes.json
+5 -1
View File
@@ -29,9 +29,10 @@ For the following examples you need to have the oCIS binary in your current work
### Using automatically generated certificates
In order to run oCIS with automatically generated and self signed certificates please execute following command. You need to replace `your-host` with an IP or hostname.
In order to run oCIS with automatically generated and self signed certificates please execute following command. You need to replace `your-host` with an IP or hostname. Since you have only self signed certificates you need to have `OCIS_INSECURE` set to `true`.
```bash
OCIS_INSECURE=true \
PROXY_HTTP_ADDR=0.0.0.0:9200 \
OCIS_URL=https://your-host:9200 \
./ocis server
@@ -42,6 +43,7 @@ OCIS_URL=https://your-host:9200 \
If you have your own certificates already in place, you may want to make oCIS use them:
```bash
OCIS_INSECURE=false \
PROXY_HTTP_ADDR=0.0.0.0:9200 \
OCIS_URL=https://your-host:9200 \
PROXY_TRANSPORT_TLS_KEY=./certs/your-host.key \
@@ -49,6 +51,8 @@ PROXY_TRANSPORT_TLS_CERT=./certs/your-host.crt \
./ocis server
```
If you generated these certificates on your own, you might need to set `OCIS_INSECURE` to `true`.
For more configuration options check the configuration section in [oCIS]({{< ref "../configuration" >}}) and the oCIS extensions.
## Start the oCIS fullstack server with Docker Compose
+2 -1
View File
@@ -45,6 +45,7 @@ In order to create the file we need first to create the folder `/etc/ocis/` and
```
OCIS_URL=https://some-hostname-or-ip:9200
PROXY_HTTP_ADDR=0.0.0.0:9200
OCIS_INSECURE=false
OCIS_LOG_LEVEL=error
@@ -56,7 +57,7 @@ PROXY_TRANSPORT_TLS_CERT=/etc/ocis/proxy/server.crt
PROXY_TRANSPORT_TLS_KEY=/etc/ocis/proxy/server.key
```
Please change your `OCIS_URL` in order to reflect your actual deployment.
Please change your `OCIS_URL` in order to reflect your actual deployment. If you are using self signed certificates you need to set `OCIS_INSECURE=true` in `/etc/ocis/ocis.env`.
## Starting the oCIS service
+1 -1
View File
@@ -98,7 +98,7 @@ git clone https://github.com/owncloud/core.git
To start ocis:
```
PROXY_ENABLE_BASIC_AUTH=true bin/ocis server
OCIS_INSECURE=true PROXY_ENABLE_BASIC_AUTH=true bin/ocis server
```
`PROXY_ENABLE_BASIC_AUTH` will allow the acceptance tests to make requests against the provisioning api (and other endpoints) using basic auth.
-26
View File
@@ -1,26 +0,0 @@
package cs3
import (
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
"google.golang.org/grpc"
)
func newConn(endpoint string) (*grpc.ClientConn, error) {
conn, err := grpc.Dial(endpoint, grpc.WithInsecure())
if err != nil {
return nil, err
}
return conn, nil
}
// GetGatewayServiceClient returns a new cs3 gateway client
func GetGatewayServiceClient(endpoint string) (gateway.GatewayAPIClient, error) {
conn, err := newConn(endpoint)
if err != nil {
return nil, err
}
return gateway.NewGatewayAPIClient(conn), nil
}
+2 -2
View File
@@ -4,9 +4,9 @@ import (
"net/http"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
"github.com/cs3org/reva/pkg/rgrpc/todo/pool"
"github.com/go-chi/chi/v5"
"github.com/owncloud/ocis/graph/pkg/config"
"github.com/owncloud/ocis/graph/pkg/cs3"
"github.com/owncloud/ocis/ocis-pkg/log"
)
@@ -24,7 +24,7 @@ func (g Graph) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// GetClient returns a gateway client to talk to reva
func (g Graph) GetClient() (gateway.GatewayAPIClient, error) {
return cs3.GetGatewayServiceClient(g.config.Reva.Address)
return pool.GetGatewayServiceClient(g.config.Reva.Address)
}
// The key type is unexported to prevent collisions with context keys defined in
+7 -2
View File
@@ -45,6 +45,11 @@ type Tracing struct {
Service string `mapstructure:"service"`
}
// Reva defines all available REVA configuration.
type Reva struct {
Address string `address`
}
// TokenManager is the config for using the reva token manager
type TokenManager struct {
JWTSecret string `mapstructure:"jwt_secret"`
@@ -67,7 +72,7 @@ type Config struct {
TokenManager TokenManager `mapstructure:"token_manager"`
Service Service `mapstructure:"service"`
AccountBackend string `mapstructure:"account_backend"`
RevaAddress string `mapstructure:"reva_address"`
Reva Reva `mapstructure:"reva"`
StorageUsersDriver string `mapstructure:"storage_users_driver"`
MachineAuthAPIKey string `mapstructure:"machine_auth_api_key"`
IdentityManagement IdentityManagement `mapstructure:"identity_management"`
@@ -116,7 +121,7 @@ func DefaultConfig() *Config {
Namespace: "com.owncloud.web",
},
AccountBackend: "accounts",
RevaAddress: "127.0.0.1:9142",
Reva: Reva{Address: "127.0.0.1:9142"},
StorageUsersDriver: "ocis",
MachineAuthAPIKey: "change-me-please",
IdentityManagement: IdentityManagement{
+3 -3
View File
@@ -4,6 +4,7 @@ import (
"net/http"
"time"
"github.com/cs3org/reva/pkg/rgrpc/todo/pool"
"github.com/owncloud/ocis/ocis-pkg/service/grpc"
"github.com/go-chi/chi/v5"
@@ -19,7 +20,6 @@ import (
ocsm "github.com/owncloud/ocis/ocs/pkg/middleware"
"github.com/owncloud/ocis/ocs/pkg/service/v0/data"
"github.com/owncloud/ocis/ocs/pkg/service/v0/response"
"github.com/owncloud/ocis/proxy/pkg/cs3"
"github.com/owncloud/ocis/proxy/pkg/user/backend"
settings "github.com/owncloud/ocis/settings/pkg/proto/v0"
)
@@ -161,9 +161,9 @@ func (o Ocs) getAccountService() accounts.AccountsService {
}
func (o Ocs) getCS3Backend() backend.UserBackend {
revaClient, err := cs3.GetGatewayServiceClient(o.config.RevaAddress)
revaClient, err := pool.GetGatewayServiceClient(o.config.Reva.Address)
if err != nil {
o.logger.Fatal().Msgf("could not get reva client at address %s", o.config.RevaAddress)
o.logger.Fatal().Msgf("could not get reva client at address %s", o.config.Reva.Address)
}
return backend.NewCS3UserBackend(nil, revaClient, o.config.MachineAuthAPIKey, o.logger)
}
+2 -2
View File
@@ -375,7 +375,7 @@ func (o Ocs) DeleteUser(w http.ResponseWriter, r *http.Request) {
return
}
if o.config.RevaAddress != "" && o.config.StorageUsersDriver != "owncloud" {
if o.config.Reva.Address != "" && o.config.StorageUsersDriver != "owncloud" {
t, err := o.mintTokenForUser(r.Context(), account)
if err != nil {
mustNotFail(render.Render(w, r, response.ErrRender(data.MetaServerError.StatusCode, errors.Wrap(err, "error minting token").Error())))
@@ -384,7 +384,7 @@ func (o Ocs) DeleteUser(w http.ResponseWriter, r *http.Request) {
ctx := metadata.AppendToOutgoingContext(r.Context(), revactx.TokenHeader, t)
gwc, err := pool.GetGatewayServiceClient(o.config.RevaAddress)
gwc, err := pool.GetGatewayServiceClient(o.config.Reva.Address)
if err != nil {
o.logger.Error().Err(err).Msg("error securing a connection to Reva gateway")
}
+3 -3
View File
@@ -171,12 +171,12 @@ func frontendConfigFromStruct(c *cli.Context, cfg *config.Config, filesCfg map[s
"prefix": cfg.Reva.Frontend.AppProviderPrefix,
"transfer_shared_secret": cfg.Reva.TransferSecret,
"timeout": 86400,
"insecure": true,
"insecure": cfg.Reva.Frontend.AppProviderInsecure,
},
"archiver": map[string]interface{}{
"prefix": cfg.Reva.Frontend.ArchiverPrefix,
"timeout": 86400,
"insecure": true,
"insecure": cfg.Reva.Frontend.ArchiverInsecure,
"max_num_files": cfg.Reva.Archiver.MaxNumFiles,
"max_size": cfg.Reva.Archiver.MaxSize,
},
@@ -191,7 +191,7 @@ func frontendConfigFromStruct(c *cli.Context, cfg *config.Config, filesCfg map[s
"files_namespace": cfg.Reva.OCDav.DavFilesNamespace,
"webdav_namespace": cfg.Reva.OCDav.WebdavNamespace,
"timeout": 86400,
"insecure": true,
"insecure": cfg.Reva.Frontend.OCDavInsecure,
"public_url": cfg.Reva.Frontend.PublicURL,
},
"ocs": map[string]interface{}{
+1 -1
View File
@@ -128,7 +128,7 @@ func storageHomeConfigFromStruct(c *cli.Context, cfg *config.Config) map[string]
"driver": cfg.Reva.StorageHome.Driver,
"drivers": storagedrivers.HomeDrivers(cfg),
"timeout": 86400,
"insecure": true,
"insecure": cfg.Reva.StorageHome.DataProvider.Insecure,
"disable_tus": false,
},
},
+1 -1
View File
@@ -150,7 +150,7 @@ func storageMetadataFromStruct(c *cli.Context, cfg *config.Config) map[string]in
"driver": cfg.Reva.StorageMetadata.Driver,
"drivers": storagedrivers.MetadataDrivers(cfg),
"timeout": 86400,
"insecure": true,
"insecure": cfg.Reva.StorageMetadata.DataProvider.Insecure,
"disable_tus": true,
},
},
+1 -1
View File
@@ -128,7 +128,7 @@ func storageUsersConfigFromStruct(c *cli.Context, cfg *config.Config) map[string
"driver": cfg.Reva.StorageUsers.Driver,
"drivers": storagedrivers.UserDrivers(cfg),
"timeout": 86400,
"insecure": true,
"insecure": cfg.Reva.StorageUsers.DataProvider.Insecure,
"disable_tus": false,
},
},
+11 -3
View File
@@ -144,10 +144,13 @@ type Groups struct {
type FrontendPort struct {
Port
AppProviderInsecure bool
AppProviderPrefix string
ArchiverInsecure bool
ArchiverPrefix string
DatagatewayPrefix string
Favorites bool
OCDavInsecure bool
OCDavPrefix string
OCSPrefix string
OCSSharePrefix string
@@ -175,6 +178,10 @@ type DataGatewayPort struct {
PublicURL string
}
type DataProvider struct {
Insecure bool
}
// StoragePort defines the available storage configuration.
type StoragePort struct {
Port
@@ -186,9 +193,10 @@ type StoragePort struct {
DataServerURL string
// for HTTP ports with only one http service
HTTPPrefix string
TempFolder string
ReadOnly bool
HTTPPrefix string
TempFolder string
ReadOnly bool
DataProvider DataProvider
}
// PublicStorage configures a public storage provider
+2 -2
View File
@@ -30,9 +30,9 @@ func AuthBearerWithConfig(cfg *config.Config) []cli.Flag {
},
&cli.BoolFlag{
Name: "oidc-insecure",
Value: flags.OverrideDefaultBool(cfg.Reva.OIDC.Insecure, true),
Value: flags.OverrideDefaultBool(cfg.Reva.OIDC.Insecure, false),
Usage: "OIDC allow insecure communication",
EnvVars: []string{"STORAGE_OIDC_INSECURE"},
EnvVars: []string{"STORAGE_OIDC_INSECURE", "OCIS_INSECURE"},
Destination: &cfg.Reva.OIDC.Insecure,
},
&cli.StringFlag{
+21
View File
@@ -119,6 +119,13 @@ func FrontendWithConfig(cfg *config.Config) []cli.Flag {
EnvVars: []string{"STORAGE_FRONTEND_APPPROVIDER_PREFIX"},
Destination: &cfg.Reva.Frontend.AppProviderPrefix,
},
&cli.BoolFlag{
Name: "approvider-insecure",
Value: flags.OverrideDefaultBool(cfg.Reva.Frontend.AppProviderInsecure, false),
Usage: "approvider insecure",
EnvVars: []string{"STORAGE_FRONTEND_APPPROVIDER_INSECURE", "OCIS_INSECURE"},
Destination: &cfg.Reva.Frontend.AppProviderInsecure,
},
&cli.StringFlag{
Name: "archiver-prefix",
Value: flags.OverrideDefaultString(cfg.Reva.Frontend.ArchiverPrefix, "archiver"),
@@ -126,6 +133,13 @@ func FrontendWithConfig(cfg *config.Config) []cli.Flag {
EnvVars: []string{"STORAGE_FRONTEND_ARCHIVER_PREFIX"},
Destination: &cfg.Reva.Frontend.ArchiverPrefix,
},
&cli.BoolFlag{
Name: "archiver-insecure",
Value: flags.OverrideDefaultBool(cfg.Reva.Frontend.ArchiverInsecure, false),
Usage: "archiver insecure",
EnvVars: []string{"STORAGE_FRONTEND_ARCHIVER_INSECURE", "OCIS_INSECURE"},
Destination: &cfg.Reva.Frontend.ArchiverInsecure,
},
&cli.StringFlag{
Name: "datagateway-prefix",
Value: flags.OverrideDefaultString(cfg.Reva.Frontend.DatagatewayPrefix, "data"),
@@ -147,6 +161,13 @@ func FrontendWithConfig(cfg *config.Config) []cli.Flag {
EnvVars: []string{"STORAGE_FRONTEND_OCDAV_PREFIX"},
Destination: &cfg.Reva.Frontend.OCDavPrefix,
},
&cli.BoolFlag{
Name: "ocdav-insecure",
Value: flags.OverrideDefaultBool(cfg.Reva.Frontend.OCDavInsecure, false),
Usage: "owncloud webdav insecure",
EnvVars: []string{"STORAGE_FRONTEND_OCDAV_INSECURE", "OCIS_INSECURE"},
Destination: &cfg.Reva.Frontend.OCDavInsecure,
},
&cli.StringFlag{
Name: "ocs-prefix",
Value: flags.OverrideDefaultString(cfg.Reva.Frontend.OCSPrefix, "ocs"),
+7
View File
@@ -130,6 +130,13 @@ func StorageHomeWithConfig(cfg *config.Config) []cli.Flag {
EnvVars: []string{"STORAGE_HOME_TMP_FOLDER"},
Destination: &cfg.Reva.StorageHome.TempFolder,
},
&cli.BoolFlag{
Name: "dataprovider-insecure",
Value: flags.OverrideDefaultBool(cfg.Reva.StorageHome.DataProvider.Insecure, false),
Usage: "dataprovider insecure",
EnvVars: []string{"STORAGE_HOME_DATAPROVIDER_INSECURE", "OCIS_INSECURE"},
Destination: &cfg.Reva.StorageHome.DataProvider.Insecure,
},
// some drivers need to look up users at the gateway
+7
View File
@@ -69,6 +69,13 @@ func StorageMetadata(cfg *config.Config) []cli.Flag {
EnvVars: []string{"STORAGE_METADATA_DRIVER"},
Destination: &cfg.Reva.StorageMetadata.Driver,
},
&cli.BoolFlag{
Name: "dataprovider-insecure",
Value: flags.OverrideDefaultBool(cfg.Reva.StorageMetadata.DataProvider.Insecure, false),
Usage: "dataprovider insecure",
EnvVars: []string{"STORAGE_METADATA_DATAPROVIDER_INSECURE", "OCIS_INSECURE"},
Destination: &cfg.Reva.StorageMetadata.DataProvider.Insecure,
},
// some drivers need to look up users at the gateway
+7
View File
@@ -78,6 +78,13 @@ func StorageUsersWithConfig(cfg *config.Config) []cli.Flag {
EnvVars: []string{"STORAGE_USERS_DRIVER"},
Destination: &cfg.Reva.StorageUsers.Driver,
},
&cli.BoolFlag{
Name: "dataprovider-insecure",
Value: flags.OverrideDefaultBool(cfg.Reva.StorageUsers.DataProvider.Insecure, false),
Usage: "dataprovider insecure",
EnvVars: []string{"STORAGE_USERS_DATAPROVIDER_INSECURE", "OCIS_INSECURE"},
Destination: &cfg.Reva.StorageUsers.DataProvider.Insecure,
},
&cli.BoolFlag{
Name: "read-only",
Value: flags.OverrideDefaultBool(cfg.Reva.StorageUsers.ReadOnly, false),
@@ -14,6 +14,7 @@ services:
WEB_UI_CONFIG: /drone/src/tests/config/drone/ocis-config.json
IDP_IDENTIFIER_REGISTRATION_CONF: /drone/src/tests/config/drone/identifier-registration.yml
ACCOUNTS_HASH_DIFFICULTY: 4
OCIS_INSECURE: "true"
# s3ng specific settings
STORAGE_USERS_DRIVER_S3NG_ENDPOINT: http://ceph:8080
STORAGE_USERS_DRIVER_S3NG_REGION: default
@@ -0,0 +1,5 @@
## Scenarios from OCIS API tests that are expected to fail with OCIS storage
#### [downloading the /Shares folder using the archiver endpoint does not work](https://github.com/owncloud/ocis/issues/2751)
- [apiArchiver/downloadById.feature:134](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiArchiver/downloadById.feature#L134)
- [apiArchiver/downloadById.feature:135](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiArchiver/downloadById.feature#L135)
@@ -11,21 +11,125 @@ Feature: download multiple resources bundled into an archive
Background:
Given user "Alice" has been created with default attributes and without skeleton files
Scenario: download a single file
Scenario Outline: download a single file
Given user "Alice" has uploaded file with content "some data" to "/textfile0.txt"
When user "Alice" downloads the archive of "/textfile0.txt" using the resource id
When user "Alice" downloads the archive of "/textfile0.txt" using the resource id and setting these headers
| header | value |
| User-Agent | <user-agent> |
Then the HTTP status code should be "200"
And the downloaded archive should contain these files:
And the downloaded <archive-type> archive should contain these files:
| name | content |
| textfile0.txt | some data |
Examples:
| user-agent | archive-type |
| Linux | tar |
| Windows NT | zip |
Scenario: download a single folder
Scenario Outline: download a single folder
Given user "Alice" has created folder "my_data"
And user "Alice" has uploaded file with content "some data" to "/my_data/textfile0.txt"
And user "Alice" has uploaded file with content "more data" to "/my_data/an_other_file.txt"
When user "Alice" downloads the archive of "/my_data" using the resource id
When user "Alice" downloads the archive of "/my_data" using the resource id and setting these headers
| header | value |
| User-Agent | <user-agent> |
Then the HTTP status code should be "200"
And the downloaded archive should contain these files:
And the downloaded <archive-type> archive should contain these files:
| name | content |
| my_data/textfile0.txt | some data |
| my_data/an_other_file.txt | more data |
Examples:
| user-agent | archive-type |
| Linux | tar |
| Windows NT | zip |
Scenario: download multiple files and folders
Given user "Alice" has uploaded file with content "some data" to "/textfile0.txt"
And user "Alice" has uploaded file with content "other data" to "/textfile1.txt"
And user "Alice" has created folder "my_data"
And user "Alice" has uploaded file with content "some data" to "/my_data/textfile2.txt"
And user "Alice" has created folder "more_data"
And user "Alice" has uploaded file with content "more data" to "/more_data/an_other_file.txt"
When user "Alice" downloads the archive of these items using the resource ids
| textfile0.txt |
| textfile1.txt |
| my_data |
| more_data |
Then the HTTP status code should be "200"
And the downloaded tar archive should contain these files:
| name | content |
| textfile0.txt | some data |
| textfile1.txt | other data |
| my_data/textfile2.txt | some data |
| more_data/an_other_file.txt | more data |
Scenario: download a single file as different user
Given user "Brian" has been created with default attributes and without skeleton files
And user "Alice" has uploaded file with content "some data" to "/textfile0.txt"
When user "Brian" downloads the archive of "/textfile0.txt" of user "Alice" using the resource id
Then the HTTP status code should be "400"
Scenario: download multiple shared items as share receiver
Given user "Brian" has been created with default attributes and without skeleton files
And user "Alice" has uploaded file with content "some data" to "/textfile0.txt"
And user "Alice" has uploaded file with content "other data" to "/textfile1.txt"
And user "Alice" has created folder "my_data"
And user "Alice" has uploaded file with content "some data" to "/my_data/textfile2.txt"
And user "Alice" has created folder "more_data"
And user "Alice" has uploaded file with content "more data" to "/more_data/an_other_file.txt"
And user "Alice" has shared file "textfile0.txt" with user "Brian"
And user "Alice" has shared file "textfile1.txt" with user "Brian"
And user "Alice" has shared folder "my_data" with user "Brian"
And user "Alice" has shared folder "more_data" with user "Brian"
And user "Brian" has accepted share "/textfile0.txt" offered by user "Alice"
And user "Brian" has accepted share "/textfile1.txt" offered by user "Alice"
And user "Brian" has accepted share "/my_data" offered by user "Alice"
And user "Brian" has accepted share "/more_data" offered by user "Alice"
When user "Brian" downloads the archive of these items using the resource ids
| /Shares/textfile0.txt |
| /Shares/textfile1.txt |
| /Shares/my_data |
| /Shares/more_data |
Then the HTTP status code should be "200"
And the downloaded tar archive should contain these files:
| name | content |
| textfile0.txt | some data |
| textfile1.txt | other data |
| my_data/textfile2.txt | some data |
| more_data/an_other_file.txt | more data |
Scenario Outline: download the Shares folder as share receiver
Given user "Brian" has been created with default attributes and without skeleton files
And user "Alice" has uploaded file with content "some data" to "/textfile0.txt"
And user "Alice" has uploaded file with content "other data" to "/textfile1.txt"
And user "Alice" has created folder "my_data"
And user "Alice" has uploaded file with content "some data" to "/my_data/textfile2.txt"
And user "Alice" has created folder "more_data"
And user "Alice" has uploaded file with content "more data" to "/more_data/an_other_file.txt"
And user "Alice" has shared file "textfile0.txt" with user "Brian"
And user "Alice" has shared file "textfile1.txt" with user "Brian"
And user "Alice" has shared folder "my_data" with user "Brian"
And user "Alice" has shared folder "more_data" with user "Brian"
And user "Brian" has accepted share "/textfile0.txt" offered by user "Alice"
And user "Brian" has accepted share "/textfile1.txt" offered by user "Alice"
And user "Brian" has accepted share "/my_data" offered by user "Alice"
And user "Brian" has accepted share "/more_data" offered by user "Alice"
When user "Brian" downloads the archive of "/Shares" using the resource id and setting these headers
| header | value |
| User-Agent | <user-agent> |
Then the HTTP status code should be "200"
And the downloaded <archive-type> archive should contain these files:
| name | content |
| Shares/textfile0.txt | some data |
| Shares/textfile1.txt | other data |
| Shares/my_data/textfile0.txt | some data |
| Shares/my_data/an_other_file.txt | more data |
Examples:
| user-agent | archive-type |
| Linux | tar |
| Windows NT | zip |
@@ -16,19 +16,16 @@ Feature: Upload files into a space
And user "Alice" lists all available spaces via the GraphApi
And user "Alice" creates a folder "mainFolder" in space "Project Venus" using the WebDav Api
Then the HTTP status code should be "201"
When user "Alice" lists the content of the space with the name "Project Venus" using the WebDav Api
Then the propfind result of the space should contain these entries:
And the space "Project Venus" should contain these entries:
| mainFolder |
Scenario: Bob creates a folder via the Graph api in a space, he expects a 404 code and
Alice checks that this folder does not exist
Scenario: Bob creates a folder via the Graph api in a space, he expects a 404 code and Alice checks that this folder does not exist
Given the administrator gives "Alice" the role "Admin" using the settings api
When user "Alice" creates a space "Project Merkur" of type "project" with quota "2000" using the GraphApi
And user "Alice" lists all available spaces via the GraphApi
And user "Bob" creates a folder "forAlice" in space "Project Merkur" using the WebDav Api
Then the HTTP status code should be "404"
When user "Alice" lists the content of the space with the name "Project Merkur" using the WebDav Api
Then the propfind result of the space should not contain these entries:
And the space "Project Merkur" should not contain these entries:
| forAlice |
Scenario: Alice creates a folder via Graph api and uploads a file
@@ -39,20 +36,17 @@ Feature: Upload files into a space
Then the HTTP status code should be "201"
And user "Alice" uploads a file inside space "Project Moon" with content "Test" to "test.txt" using the WebDAV API
Then the HTTP status code should be "201"
When user "Alice" lists the content of the space with the name "Project Moon" using the WebDav Api
Then the propfind result of the space should contain these entries:
And the space "Project Moon" should contain these entries:
| NewFolder |
| test.txt |
Scenario: Bob uploads a file via the Graph api in a space, he expects a 404 code and
Alice checks that this file does not exist
Scenario: Bob uploads a file via the Graph api in a space, he expects a 404 code and Alice checks that this file does not exist
Given the administrator gives "Alice" the role "Admin" using the settings api
When user "Alice" creates a space "Project Pluto" of type "project" with quota "2000" using the GraphApi
And user "Alice" lists all available spaces via the GraphApi
And user "Bob" uploads a file inside space "Project Pluto" with content "Test" to "test.txt" using the WebDAV API
Then the HTTP status code should be "404"
When user "Alice" lists the content of the space with the name "Project Pluto" using the WebDav Api
Then the propfind result of the space should not contain these entries:
And the space "Project Pluto" should not contain these entries:
| test.txt |
Scenario: Alice creates uploads a file and checks her quota
@@ -63,16 +63,29 @@ class ArchiverContext implements Context {
}
/**
* @When user :user downloads the archive of :resourceId using the resource id
* @When user :user downloads the archive of :resourceId using the resource id and setting these headers
*
* @param string $user
* @param string $resource
* @param TableNode $headersTable
*
* @return void
*
* @throws \GuzzleHttp\Exception\GuzzleException
*/
public function userDownloadsTheArchiveOfUsingTheResourceId(string $user, string $resource): void {
public function userDownloadsTheArchiveOfUsingTheResourceId(
string $user,
string $resource,
TableNode $headersTable
): void {
$this->featureContext->verifyTableNodeColumns(
$headersTable,
['header', 'value']
);
$headers = [];
foreach ($headersTable as $row) {
$headers[$row['header']] = $row ['value'];
}
$resourceId = $this->featureContext->getFileIdForPath($user, $resource);
$user = $this->featureContext->getActualUsername($user);
$this->featureContext->setResponse(
@@ -80,25 +93,86 @@ class ArchiverContext implements Context {
$this->featureContext->getBaseUrl() . '/archiver?id=' . $resourceId,
'',
$user,
$this->featureContext->getPasswordForUser($user)
$this->featureContext->getPasswordForUser($user),
$headers
)
);
}
/**
* @Then the downloaded archive should contain these files:
* @When user :downloader downloads the archive of :item of user :owner using the resource id
*
* @param string $downloader Who sends the request
* @param string $resource
* @param string $owner Who is the real owner of the file
*
* @return void
*
* @throws \GuzzleHttp\Exception\GuzzleException
*/
public function userDownloadsTheArchiveOfItemOfUserUsingTheResourceId(
string $downloader,
string $resource,
string $owner
): void {
$resourceId = $this->featureContext->getFileIdForPath($owner, $resource);
$downloader = $this->featureContext->getActualUsername($downloader);
$this->featureContext->setResponse(
HttpRequestHelper::get(
$this->featureContext->getBaseUrl() . '/archiver?id=' . $resourceId,
'',
$downloader,
$this->featureContext->getPasswordForUser($downloader),
)
);
}
/**
* @When user :arg1 downloads the archive of these items using the resource ids
*
* @param string $user
* @param TableNode $items
*
* @return void
*
* @throws \GuzzleHttp\Exception\GuzzleException
*/
public function userDownloadsTheArchiveOfTheseItemsUsingTheResourceIds(
string $user,
TableNode $items
): void {
$user = $this->featureContext->getActualUsername($user);
$resourceIdsString = '';
foreach ($items->getRows() as $item) {
$fileId = $this->featureContext->getFileIdForPath($user, $item[0]);
$resourceIdsString .= 'id=' . $fileId . '&';
}
$resourceIdsString = \rtrim($resourceIdsString, '&');
$this->featureContext->setResponse(
HttpRequestHelper::get(
$this->featureContext->getBaseUrl() . '/archiver?' . $resourceIdsString,
'',
$user,
$this->featureContext->getPasswordForUser($user),
)
);
}
/**
* @Then the downloaded :type archive should contain these files:
*
* @param string $type
* @param TableNode $expectedFiles
*
* @return void
*
* @throws Exception
*/
public function theDownloadedArchiveShouldContainTheseFiles(TableNode $expectedFiles) {
public function theDownloadedArchiveShouldContainTheseFiles(string $type, TableNode $expectedFiles) {
$this->featureContext->verifyTableNodeColumns($expectedFiles, ['name', 'content']);
$tempFile = \tempnam(\sys_get_temp_dir(), 'OcAcceptanceTests_');
\unlink($tempFile); // we only need the name
$tempFile = $tempFile . '.tar'; // it needs the extension
$tempFile = $tempFile . '.' . $type; // it needs the extension
\file_put_contents($tempFile, $this->featureContext->getResponse()->getBody()->getContents());
$archive = UnifiedArchive::open($tempFile);
foreach ($expectedFiles->getHash() as $expectedFile) {
File diff suppressed because it is too large Load Diff
+2
View File
@@ -61,6 +61,7 @@ type Thumbnail struct {
Resolutions []string `mapstructure:"resolutions"`
FileSystemStorage FileSystemStorage `mapstructure:"filesystem_storage"`
WebdavAllowInsecure bool `mapstructure:"webdav_allow_insecure"`
CS3AllowInsecure bool `mapstructure:"cs3_allow_insecure"`
RevaGateway string `mapstructure:"reva_gateway"`
WebdavNamespace string `mapstructure:"webdav_namespace"`
}
@@ -99,6 +100,7 @@ func DefaultConfig() *Config {
WebdavAllowInsecure: true,
RevaGateway: "127.0.0.1:9142",
WebdavNamespace: "/home",
CS3AllowInsecure: false,
},
}
}
+1 -1
View File
@@ -42,7 +42,7 @@ func NewService(opts ...Option) grpc.Service {
options.Logger,
),
),
svc.CS3Source(imgsource.NewCS3Source(gc)),
svc.CS3Source(imgsource.NewCS3Source(tconf, gc)),
svc.CS3Client(gc),
)
thumbnail = svc.NewInstrument(thumbnail, options.Metrics)
+9 -4
View File
@@ -12,6 +12,7 @@ import (
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
revactx "github.com/cs3org/reva/pkg/ctx"
"github.com/cs3org/reva/pkg/rhttp"
"github.com/owncloud/ocis/thumbnails/pkg/config"
"github.com/pkg/errors"
"google.golang.org/grpc/metadata"
)
@@ -23,12 +24,14 @@ const (
)
type CS3 struct {
client gateway.GatewayAPIClient
client gateway.GatewayAPIClient
insecure bool
}
func NewCS3Source(c gateway.GatewayAPIClient) CS3 {
func NewCS3Source(cfg config.Thumbnail, c gateway.GatewayAPIClient) CS3 {
return CS3{
client: c,
client: c,
insecure: cfg.CS3AllowInsecure,
}
}
@@ -67,7 +70,9 @@ func (s CS3) Get(ctx context.Context, path string) (io.ReadCloser, error) {
httpReq.Header.Set(revactx.TokenHeader, auth)
httpReq.Header.Set(TokenTransportHeader, tk)
http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec
http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{
InsecureSkipVerify: s.insecure, //nolint:gosec
}
client := &http.Client{}
resp, err := client.Do(httpReq) // nolint:bodyclose