Merge pull request #6014 from owncloud/ainmosni/feature/gdpr-collection

Integrate keycloak and events data into graph.

ocis-pkg/keycloak/client.go

@@ -116,8 +116,8 @@ func (c *ConcreteClient) GetUserByEmail(ctx context.Context, realm, mail string)
 }
 
 // GetPIIReport returns a structure with all the PII for the user.
-func (c *ConcreteClient) GetPIIReport(ctx context.Context, realm string, user *libregraph.User) (*PIIReport, error) {
-	u, err := c.GetUserByEmail(ctx, realm, *user.Mail)
+func (c *ConcreteClient) GetPIIReport(ctx context.Context, realm string, email string) (*PIIReport, error) {
+	u, err := c.GetUserByEmail(ctx, realm, email)
 	if err != nil {
 		return nil, err
 	}
@@ -164,17 +164,20 @@ func (c *ConcreteClient) getToken(ctx context.Context) (*gocloak.JWT, error) {
 }
 
 func (c *ConcreteClient) keycloakUserToLibregraph(u *gocloak.User) *libregraph.User {
-	attrs := *u.Attributes
-	ldapID := ""
-	ldapIDs, ok := attrs[_idAttr]
-	if ok {
-		ldapID = ldapIDs[0]
-	}
-
+	var ldapID string
 	var userType *string
-	userTypes, ok := attrs[_userTypeAttr]
-	if ok {
-		userType = &userTypes[0]
+
+	if u.Attributes != nil {
+		attrs := *u.Attributes
+		ldapIDs, ok := attrs[_idAttr]
+		if ok {
+			ldapID = ldapIDs[0]
+		}
+
+		userTypes, ok := attrs[_userTypeAttr]
+		if ok {
+			userType = &userTypes[0]
+		}
 	}
 
 	return &libregraph.User{

ocis-pkg/keycloak/types.go

@@ -26,8 +26,8 @@ var userActionsToString = map[UserAction]string{
 
 // PIIReport is a structure of all the PersonalIdentifiableInformation contained in keycloak.
 type PIIReport struct {
-	UserData    *libregraph.User
-	Credentials []*gocloak.CredentialRepresentation
+	UserData    *libregraph.User                    `json:"user_data,omitempty"`
+	Credentials []*gocloak.CredentialRepresentation `json:"credentials,omitempty"`
 }
 
 // Client represents a keycloak client.
@@ -35,5 +35,5 @@ type Client interface {
 	CreateUser(ctx context.Context, realm string, user *libregraph.User, userActions []UserAction) (string, error)
 	SendActionsMail(ctx context.Context, realm, userID string, userActions []UserAction) error
 	GetUserByEmail(ctx context.Context, realm, email string) (*libregraph.User, error)
-	GetPIIReport(ctx context.Context, realm string, user *libregraph.User) (*PIIReport, error)
+	GetPIIReport(ctx context.Context, realm string, email string) (*PIIReport, error)
 }

services/graph/README.md

@@ -30,3 +30,27 @@ The `graph` service can use a configured store via `GRAPH_STORE_TYPE`. Possible
 2.  Though usually not necessary, a database name and a database table can be configured for event stores if the event store supports this. Generally not applicapable for stores of type `in-memory`. These settings are blank by default which means that the standard settings of the configured store applies.
 3.  The graph service can be scaled if not using `in-memory` stores and the stores are configured identically over all instances.
 4.  When using `redis-sentinel`, the Redis master to use is configured via `GRAPH_CACHE_STORE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
+
+## Keycloak Configuration For The Personal Data Export
+
+If Keycloak is used for authentication, GDPR regulations require to add all personal identifiable information that Keycloak has about the user to the personal data export. To do this, the following environment variables must be set:
+
+*   `OCIS_KEYCLOAK_BASE_PATH` - The URL to the keycloak instance.
+*   `OCIS_KEYCLOAK_CLIENT_ID` - The client ID of the client that is used to authenticate with keycloak, this client has to be able to list users and get the credential data. 
+*   `OCIS_KEYCLOAK_CLIENT_SECRET` - The client secret of the client that is used to authenticate with keycloak.
+*   `OCIS_KEYCLOAK_CLIENT_REALM` - The realm the client is defined in.
+*   `OCIS_KEYCLOAK_USER_REALM` - The realm the oCIS users are defined in.
+*   `OCIS_KEYCLOAK_INSECURE_SKIP_VERIFY` - If set to true, the TLS certificate of the keycloak instance is not verified.
+
+### Keycloak Client Configuration
+
+The client that is used to authenticate with keycloak has to be able to list users and get the credential data. To do this, the following  roles have to be assigned to the client and they have to be about the realm that contains the oCIS users:
+
+*   `view-users`
+*   `view-identity-providers`
+*   `view-realm`
+*   `view-clients`
+*   `view-events`
+*   `view-authorization`
+
+Note that these roles are only available to assign if the client is in the `master` realm.

services/graph/pkg/config/config.go

@@ -30,7 +30,8 @@ type Config struct {
 	Identity    Identity    `yaml:"identity"`
 	Events      Events      `yaml:"events"`
 
-	MachineAuthAPIKey string `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY;USERLOG_MACHINE_AUTH_API_KEY" desc:"Machine auth API key used to validate internal requests necessary to access resources from other services."`
+	MachineAuthAPIKey string   `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY;USERLOG_MACHINE_AUTH_API_KEY" desc:"Machine auth API key used to validate internal requests necessary to access resources from other services."`
+	Keycloak          Keycloak `yaml:"keycloak"`
 
 	Context context.Context `yaml:"-"`
 }
@@ -121,3 +122,13 @@ type CORS struct {
 	AllowedHeaders   []string `yaml:"allow_headers" env:"OCIS_CORS_ALLOW_HEADERS;GRAPH_CORS_ALLOW_HEADERS" desc:"A comma-separated list of allowed CORS headers. See following chapter for more details: *Access-Control-Request-Headers* at https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Request-Headers."`
 	AllowCredentials bool     `yaml:"allow_credentials" env:"OCIS_CORS_ALLOW_CREDENTIALS;GRAPH_CORS_ALLOW_CREDENTIALS" desc:"Allow credentials for CORS.See following chapter for more details: *Access-Control-Allow-Credentials* at https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Credentials."`
 }
+
+// Keycloak configuration
+type Keycloak struct {
+	BasePath           string `yaml:"base_path" env:"OCIS_KEYCLOAK_BASE_PATH;GRAPH_KEYCLOAK_BASE_PATH" desc:"The URL to access keycloak."`
+	ClientID           string `yaml:"client_id" env:"OCIS_KEYCLOAK_CLIENT_ID;GRAPH_KEYCLOAK_CLIENT_ID" desc:"The client id to authenticate with keycloak."`
+	ClientSecret       string `yaml:"client_secret" env:"OCIS_KEYCLOAK_CLIENT_SECRET;GRAPH_KEYCLOAK_CLIENT_SECRET" desc:"The client secret to use in authentication."`
+	ClientRealm        string `yaml:"client_realm" env:"OCIS_KEYCLOAK_CLIENT_REALM;GRAPH_KEYCLOAK_CLIENT_REALM" desc:"The realm the client is defined in."`
+	UserRealm          string `yaml:"user_realm" env:"OCIS_KEYCLOAK_USER_REALM;GRAPH_KEYCLOAK_USER_REALM" desc:"The realm users are defined."`
+	InsecureSkipVerify bool   `yaml:"insecure_skip_verify" env:"OCIS_KEYCLOAK_INSECURE_SKIP_VERIFY;GRAPH_KEYCLOAK_INSECURE_SKIP_VERIFY" desc:"Disable TLS certificate validation for Keycloak connections. Do not set this in production environments."`
+}

services/graph/pkg/server/http/server.go

@@ -15,10 +15,13 @@ import (
 	"github.com/owncloud/ocis/v2/ocis-pkg/account"
 	"github.com/owncloud/ocis/v2/ocis-pkg/cors"
 	ociscrypto "github.com/owncloud/ocis/v2/ocis-pkg/crypto"
+	"github.com/owncloud/ocis/v2/ocis-pkg/keycloak"
 	"github.com/owncloud/ocis/v2/ocis-pkg/middleware"
 	"github.com/owncloud/ocis/v2/ocis-pkg/service/grpc"
+	ogrpc "github.com/owncloud/ocis/v2/ocis-pkg/service/grpc"
 	"github.com/owncloud/ocis/v2/ocis-pkg/service/http"
 	"github.com/owncloud/ocis/v2/ocis-pkg/version"
+	ehsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/eventhistory/v0"
 	searchsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/search/v0"
 	settingssvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/settings/v0"
 	graphMiddleware "github.com/owncloud/ocis/v2/services/graph/pkg/middleware"
@@ -132,6 +135,18 @@ func Server(opts ...Option) (http.Service, error) {
 		// no gatewayclient needed
 	}
 
+	// Keycloak client is optional, so if it stays nil, it's fine.
+	var keyCloakClient keycloak.Client
+	if options.Config.Keycloak.BasePath != "" {
+		kcc := options.Config.Keycloak
+		if kcc.ClientID == "" || kcc.ClientSecret == "" || kcc.ClientRealm == "" || kcc.UserRealm == "" {
+			return http.Service{}, errors.New("keycloak client id, secret, client realm and user realm must be set")
+		}
+		keyCloakClient = keycloak.New(kcc.BasePath, kcc.ClientID, kcc.ClientSecret, kcc.ClientRealm, kcc.InsecureSkipVerify)
+	}
+
+	hClient := ehsvc.NewEventHistoryService("com.owncloud.api.eventhistory", ogrpc.DefaultClient())
+
 	var handle svc.Service
 	handle, err = svc.NewService(
 		svc.Logger(options.Logger),
@@ -142,6 +157,8 @@ func Server(opts ...Option) (http.Service, error) {
 		svc.WithRequireAdminMiddleware(requireAdminMiddleware),
 		svc.WithGatewayClient(gatewayClient),
 		svc.WithSearchService(searchsvc.NewSearchProviderService("com.owncloud.api.search", grpc.DefaultClient())),
+		svc.KeycloakClient(keyCloakClient),
+		svc.EventHistoryClient(hClient),
 	)
 
 	if err != nil {

services/graph/pkg/service/v0/graph.go

@@ -13,7 +13,9 @@ import (
 	"github.com/go-chi/chi/v5"
 	"github.com/jellydator/ttlcache/v3"
 	libregraph "github.com/owncloud/libre-graph-api-go"
+	"github.com/owncloud/ocis/v2/ocis-pkg/keycloak"
 	"github.com/owncloud/ocis/v2/ocis-pkg/log"
+	ehsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/eventhistory/v0"
 	searchsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/search/v0"
 	settingssvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/settings/v0"
 	"github.com/owncloud/ocis/v2/services/graph/pkg/config"
@@ -68,6 +70,8 @@ type Graph struct {
 	groupsCache              *ttlcache.Cache[string, libregraph.Group]
 	eventsPublisher          events.Publisher
 	searchService            searchsvc.SearchProviderService
+	keycloakClient           keycloak.Client
+	historyClient            ehsvc.EventHistoryService
 }
 
 // ServeHTTP implements the Service interface.

services/graph/pkg/service/v0/option.go

@@ -5,8 +5,10 @@ import (
 
 	gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
 	"github.com/cs3org/reva/v2/pkg/events"
+	"github.com/owncloud/ocis/v2/ocis-pkg/keycloak"
 	"github.com/owncloud/ocis/v2/ocis-pkg/log"
 	"github.com/owncloud/ocis/v2/ocis-pkg/roles"
+	ehsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/eventhistory/v0"
 	searchsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/search/v0"
 	settingssvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/settings/v0"
 	"github.com/owncloud/ocis/v2/services/graph/pkg/config"
@@ -30,6 +32,8 @@ type Options struct {
 	RoleManager              *roles.Manager
 	EventsPublisher          events.Publisher
 	SearchService            searchsvc.SearchProviderService
+	KeycloakClient           keycloak.Client
+	EventHistoryClient       ehsvc.EventHistoryService
 }
 
 // newOptions initializes the available default options.
@@ -126,3 +130,17 @@ func EventsPublisher(val events.Publisher) Option {
 		o.EventsPublisher = val
 	}
 }
+
+// KeycloakClient provides a function to set the KeycloakCient option.
+func KeycloakClient(val keycloak.Client) Option {
+	return func(o *Options) {
+		o.KeycloakClient = val
+	}
+}
+
+// EventHistoryClient provides a function to set the EventHistoryClient option.
+func EventHistoryClient(val ehsvc.EventHistoryService) Option {
+	return func(o *Options) {
+		o.EventHistoryClient = val
+	}
+}

services/graph/pkg/service/v0/personaldata.go

@@ -19,6 +19,8 @@ import (
 	"github.com/cs3org/reva/v2/pkg/events"
 	"github.com/cs3org/reva/v2/pkg/rhttp"
 	"github.com/cs3org/reva/v2/pkg/utils"
+	ehmsg "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/eventhistory/v0"
+	ehsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/eventhistory/v0"
 )
 
 var (
@@ -75,12 +77,35 @@ func (g Graph) ExportPersonalData(w http.ResponseWriter, r *http.Request) {
 
 // GatherPersonalData will all gather all personal data of the user and save it to a file in the users personal space
 func (g Graph) GatherPersonalData(usr *user.User, ref *provider.Reference, token string, marsh Marshaller) {
+	// the context might already be cancelled. We need to impersonate the acting user again
+	ctx, err := utils.ImpersonateUser(usr, g.gatewayClient, g.config.MachineAuthAPIKey)
+	if err != nil {
+		g.logger.Error().Err(err).Str("userID", usr.GetId().GetOpaqueId()).Msg("cannot impersonate user")
+	}
 	// create data
 	data := make(map[string]interface{})
 
 	// reva user
 	data["user"] = usr
 
+	// Check if we have a keycloak client, and if so, get the keycloak export.
+	if ctx != nil && g.keycloakClient != nil {
+		kcd, err := g.keycloakClient.GetPIIReport(ctx, g.config.Keycloak.UserRealm, usr.GetMail())
+		if err != nil {
+			g.logger.Error().Err(err).Str("userID", usr.GetId().GetOpaqueId()).Msg("cannot get keycloak personal data")
+		}
+		data["keycloak"] = kcd
+	}
+
+	// get event data
+	if ctx != nil && g.historyClient != nil {
+		resp, err := g.historyClient.GetEventsForUser(ctx, &ehsvc.GetEventsForUserRequest{UserID: usr.GetId().GetOpaqueId()})
+		if err != nil {
+			g.logger.Error().Err(err).Str("userID", usr.GetId().GetOpaqueId()).Msg("cannot get event personal data")
+		}
+		data["events"] = convertEvents(resp.GetEvents())
+	}
+
 	// marshal
 	by, err := marsh(data)
 	if err != nil {
@@ -198,7 +223,6 @@ func createFolders(ctx context.Context, ref *provider.Reference, gwc gateway.Gat
 		}
 	}
 	return nil
-
 }
 
 func getLocation(r *http.Request) string {
@@ -214,3 +238,18 @@ func getLocation(r *http.Request) string {
 
 	return _backupFileName
 }
+
+// we want the events to look nice in the file, don't we?
+func convertEvents(evs []*ehmsg.Event) []map[string]interface{} {
+	out := make([]map[string]interface{}, len(evs))
+	for i, e := range evs {
+		var content map[string]interface{}
+		_ = json.Unmarshal(e.GetEvent(), &content)
+		out[i] = map[string]interface{}{
+			"id":    e.GetId(),
+			"type":  e.GetType(),
+			"event": content,
+		}
+	}
+	return out
+}

services/graph/pkg/service/v0/service.go

@@ -143,6 +143,8 @@ func NewService(opts ...Option) (Graph, error) {
 		gatewayClient:            options.GatewayClient,
 		searchService:            options.SearchService,
 		identityEducationBackend: options.IdentityEducationBackend,
+		keycloakClient:           options.KeycloakClient,
+		historyClient:            options.EventHistoryClient,
 	}
 
 	if err := setIdentityBackends(options, &svc); err != nil {

services/invitations/pkg/config/config.go

@@ -26,10 +26,10 @@ type Config struct {
 
 // Keycloak configuration
 type Keycloak struct {
-	BasePath           string `yaml:"base_path" env:"INVITATIONS_KEYCLOAK_BASE_PATH" desc:"The URL to access keycloak."`
-	ClientID           string `yaml:"client_id" env:"INVITATIONS_KEYCLOAK_CLIENT_ID" desc:"The client id to authenticate with keycloak."`
-	ClientSecret       string `yaml:"client_secret" env:"INVITATIONS_KEYCLOAK_CLIENT_SECRET" desc:"The client secret to use in authentication."`
-	ClientRealm        string `yaml:"client_realm" env:"INVITATIONS_KEYCLOAK_CLIENT_REALM" desc:"The realm the client is defined in."`
-	UserRealm          string `yaml:"user_realm" env:"INVITATIONS_KEYCLOAK_USER_REALM" desc:"The realm users are defined."`
-	InsecureSkipVerify bool   `yaml:"insecure_skip_verify" env:"INVITATIONS_KEYCLOAK_INSECURE_SKIP_VERIFY" desc:"Disable TLS certificate validation for Keycloak connections. Do not set this in production environments."`
+	BasePath           string `yaml:"base_path" env:"OCIS_KEYCLOAK_BASE_PATH;INVITATIONS_KEYCLOAK_BASE_PATH" desc:"The URL to access keycloak."`
+	ClientID           string `yaml:"client_id" env:"OCIS_KEYCLOAK_CLIENT_ID;INVITATIONS_KEYCLOAK_CLIENT_ID" desc:"The client id to authenticate with keycloak."`
+	ClientSecret       string `yaml:"client_secret" env:"OCIS_KEYCLOAK_CLIENT_SECRET;INVITATIONS_KEYCLOAK_CLIENT_SECRET" desc:"The client secret to use in authentication."`
+	ClientRealm        string `yaml:"client_realm" env:"OCIS_KEYCLOAK_CLIENT_REALM;INVITATIONS_KEYCLOAK_CLIENT_REALM" desc:"The realm the client is defined in."`
+	UserRealm          string `yaml:"user_realm" env:"OCIS_KEYCLOAK_USER_REALM;INVITATIONS_KEYCLOAK_USER_REALM" desc:"The realm users are defined."`
+	InsecureSkipVerify bool   `yaml:"insecure_skip_verify" env:"OCIS_KEYCLOAK_INSECURE_SKIP_VERIFY;INVITATIONS_KEYCLOAK_INSECURE_SKIP_VERIFY" desc:"Disable TLS certificate validation for Keycloak connections. Do not set this in production environments."`
 }

services/invitations/pkg/mocks/client.go

@@ -40,25 +40,25 @@ func (_m *Client) CreateUser(ctx context.Context, realm string, user *libregraph
 	return r0, r1
 }
 
-// GetPIIReport provides a mock function with given fields: ctx, realm, user
-func (_m *Client) GetPIIReport(ctx context.Context, realm string, user *libregraph.User) (*keycloak.PIIReport, error) {
-	ret := _m.Called(ctx, realm, user)
+// GetPIIReport provides a mock function with given fields: ctx, realm, email
+func (_m *Client) GetPIIReport(ctx context.Context, realm string, email string) (*keycloak.PIIReport, error) {
+	ret := _m.Called(ctx, realm, email)
 
 	var r0 *keycloak.PIIReport
 	var r1 error
-	if rf, ok := ret.Get(0).(func(context.Context, string, *libregraph.User) (*keycloak.PIIReport, error)); ok {
-		return rf(ctx, realm, user)
+	if rf, ok := ret.Get(0).(func(context.Context, string, string) (*keycloak.PIIReport, error)); ok {
+		return rf(ctx, realm, email)
 	}
-	if rf, ok := ret.Get(0).(func(context.Context, string, *libregraph.User) *keycloak.PIIReport); ok {
-		r0 = rf(ctx, realm, user)
+	if rf, ok := ret.Get(0).(func(context.Context, string, string) *keycloak.PIIReport); ok {
+		r0 = rf(ctx, realm, email)
 	} else {
 		if ret.Get(0) != nil {
 			r0 = ret.Get(0).(*keycloak.PIIReport)
 		}
 	}
 
-	if rf, ok := ret.Get(1).(func(context.Context, string, *libregraph.User) error); ok {
-		r1 = rf(ctx, realm, user)
+	if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok {
+		r1 = rf(ctx, realm, email)
 	} else {
 		r1 = ret.Error(1)
 	}