From 5cc98f0792f47878dfbbc691581a2a7329e22479 Mon Sep 17 00:00:00 2001 From: Pascal Bleser Date: Wed, 24 Sep 2025 16:04:51 +0200 Subject: [PATCH] Docker Compose Groupware improvements * made a few changes in order to further simplify the setup for developers of the Groupware backend * add STALWART_DOMAIN to deployments/examples/opencloud_full/.env * adapt the Stalwart configuration file to not set server.hostname and, instead, pick it up from /etc/hostname, which is set by Docker Compose as we can use default values for STALWART_DOMAIN there, in an analogous fashion to the other containers in that project * add config/keycloak/clients/groupware.json to avoid requiring manual configuration of Keycloak via the admin web UI * Stalwart container: - listen for SMTPS on :1465 - remove the stalwart-logs volume, not needed (logs are going to stdout) * updated services/groupware/DEVELOPER.md: - refer to a variable OCDIR to make instructions more copy-pasteable - remove manual Keycloak configuration section as it is now obsolete, replaced by provisioning a configuration file instead --- devtools/deployments/opencloud_full/.env | 3 + .../config/keycloak/clients/groupware.json | 58 ++++++++++++ .../config/stalwart/config.toml | 1 - .../deployments/opencloud_full/stalwart.yml | 4 +- services/groupware/DEVELOPER.md | 91 ++++++++----------- 5 files changed, 101 insertions(+), 56 deletions(-) create mode 100644 devtools/deployments/opencloud_full/config/keycloak/clients/groupware.json diff --git a/devtools/deployments/opencloud_full/.env b/devtools/deployments/opencloud_full/.env index c06402da6..39e940fab 100644 --- a/devtools/deployments/opencloud_full/.env +++ b/devtools/deployments/opencloud_full/.env @@ -308,6 +308,9 @@ KEYCLOAK_ADMIN_PASSWORD= ### Stalwart Settings ### # Note: the leading colon is required to enable the service. #STALWART=:stalwart.yml +# Domain of Stalwart +# Defaults to "stalwart.opencloud.test" +STALWART_DOMAIN= ## IMPORTANT ## # This MUST be the last line as it assembles the supplemental compose files to be used. diff --git a/devtools/deployments/opencloud_full/config/keycloak/clients/groupware.json b/devtools/deployments/opencloud_full/config/keycloak/clients/groupware.json new file mode 100644 index 000000000..775bc03ea --- /dev/null +++ b/devtools/deployments/opencloud_full/config/keycloak/clients/groupware.json @@ -0,0 +1,58 @@ +{ + "clientId": "groupware", + "name": "OpenCloud Groupware", + "description": "Used for authenticating automated HTTP clients of the OpenCloud Groupware API", + "rootUrl": "", + "adminUrl": "", + "baseUrl": "", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "/*" + ], + "webOrigins": [ + "/*" + ], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": true, + "protocol": "openid-connect", + "attributes": { + "oidc.ciba.grant.enabled": "false", + "backchannel.logout.session.required": "true", + "oauth2.device.authorization.grant.enabled": "false", + "backchannel.logout.revoke.offline.tokens": "false" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": true, + "nodeReRegistrationTimeout": -1, + "defaultClientScopes": [ + "web-origins", + "acr", + "profile", + "roles", + "groups", + "OpenCloudUnique_ID", + "basic", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ], + "access": { + "view": true, + "configure": true, + "manage": true + } +} diff --git a/devtools/deployments/opencloud_full/config/stalwart/config.toml b/devtools/deployments/opencloud_full/config/stalwart/config.toml index 44c452e90..e17e01b30 100644 --- a/devtools/deployments/opencloud_full/config/stalwart/config.toml +++ b/devtools/deployments/opencloud_full/config/stalwart/config.toml @@ -38,7 +38,6 @@ directory.ldap.url = "ldap://ldap-server:1389" metrics.prometheus.auth.secret = "secret" metrics.prometheus.auth.username = "metrics" metrics.prometheus.enable = true -server.hostname = "stalwart.opencloud.test" http.allowed-endpoint = 200 http.hsts = true http.permissive-cors = false diff --git a/devtools/deployments/opencloud_full/stalwart.yml b/devtools/deployments/opencloud_full/stalwart.yml index d3d240781..e4d376894 100644 --- a/devtools/deployments/opencloud_full/stalwart.yml +++ b/devtools/deployments/opencloud_full/stalwart.yml @@ -8,16 +8,17 @@ services: stalwart: image: ghcr.io/stalwartlabs/stalwart:v0.13.2-alpine + hostname: ${STALWART_DOMAIN:-stalwart.opencloud.test} networks: - opencloud-net ports: - "127.0.0.1:143:143" - "127.0.0.1:993:993" + - "127.0.0.1:1465:465" volumes: - /etc/localtime:/etc/localtime:ro - ./config/stalwart:/opt/stalwart/etc - stalwart-data:/opt/stalwart/data - - stalwart-logs:/opt/stalwart/logs labels: - "traefik.enable=true" - "traefik.http.routers.stalwart.entrypoints=https" @@ -31,7 +32,6 @@ services: volumes: stalwart-data: - stalwart-logs: networks: opencloud-net: diff --git a/services/groupware/DEVELOPER.md b/services/groupware/DEVELOPER.md index 2d74f4a4d..157310ad8 100644 --- a/services/groupware/DEVELOPER.md +++ b/services/groupware/DEVELOPER.md @@ -15,15 +15,19 @@ Use [the `groupware` branch](https://github.com/opencloud-eu/opencloud/tree/grou ```bash cd ~/src/opencloud/ +OCDIR="$PWD" git clone --branch groupware git@github.com:opencloud-eu/opencloud.git ``` +Note that setting the variable `OCDIR` is merely going to help us with keeping the instructions below as generic as possible, it is not an environment variable that is used by OpenCloud. + + Also, you might want to check out these [helper scripts in opencloud-tools](https://github.com/pbleser-oc/opencloud-tools) somewhere and put that directory into your `PATH`, as it contains scripts to test and build the OpenCloud Groupware: ```bash -cd ~/src/opencloud/ +cd "$OCDIR/" git clone git@github.com:pbleser-oc/opencloud-tools.git ./bin -echo 'export PATH="$PATH:$HOME/src/opencloud/bin"' >> ~/.bashrc +echo "export PATH=\"\$PATH:$OCDIR/bin\"" >> ~/.bashrc ``` Those scripts have the following prerequisites: @@ -37,6 +41,8 @@ Since we require having a Stalwart container running at the very least, the pref ## Configuration +The default hostname domain for the containers is `.opencloud.test` + ### Hosts Make sure to have the following entries in your `/etc/hosts`: @@ -52,9 +58,15 @@ Make sure to have the following entries in your `/etc/hosts`: 127.0.0.1 traefik.opencloud.test ``` +Alternatively, use the following shell snippet to extract it in a more automated fashion: + +```bash +perl -ne 'if (/^([A-Z][A-Z0-9]+)_DOMAIN=(.*)$/) { print length($2) < 1 ? lc($1).".opencloud.test" : $2,"\n"}' <.env|sort|while read n; do grep -w -q "$n" /etc/hosts && echo -e "\e[32;4mexists :\e[0m $n: \e[32m$(grep -w $n /etc/hosts)\e[0m">&2 || { echo -e "\e[33;4mmissing:\e[0m ${n}" >&2; echo -e "127.0.0.1\t${n}";}; done | sudo tee -a /etc/hosts +``` + ### Compose -It first needs to be tuned a little, and for that, edit `deployments/examples/opencloud_full/.env`, making the following changes: +It first needs to be tuned a little, and for that, edit `$OCDIR/opencloud/deployments/examples/opencloud_full/.env`, making the following changes: * change the container image to `opencloudeu/opencloud:dev`: ```diff @@ -113,24 +125,25 @@ It first needs to be tuned a little, and for that, edit `deployments/examples/op Build the `opencloudeu/opencloud:dev` image first: ```bash -cd ~/src/opencloud/opencloud -make -C opencloud/ clean build dev-docker +cd "$OCDIR/opencloud/" +make -C ./opencloud/ clean build dev-docker ``` If you see obscure JavaScript related errors, do this and then try the `make` command above again: ```bash -make -C opencloud/services/idp/ generate +make -C ./opencloud/services/idp/ generate +make -C ./opencloud/ clean build dev-docker ``` And then either run everything from the Docker Compose `opencloud_full` setup: ```bash -cd ./deployments/examples/opencloud_full/ +cd "$OCDIR/opencloud/deployments/examples/opencloud_full/" docker compose up -d ``` -or from within VSCode, in which case you should run all the services from the Docker Compose setup as above, but stop the `opencloud` service container (as that one will be running from within your IDE instead): +or, if you plan to make changes to the backend code base, it might be more convenient to do so from within VSCode, in which case you should run all the services from the Docker Compose setup as above, but stop the `opencloud` service container (as that one will be running from within your IDE instead): ```bash docker stop opencloud_full-opencloud-1 @@ -138,37 +151,7 @@ docker stop opencloud_full-opencloud-1 and then use the Launcher `OpenCloud server with external services` in VSCode. -## Keycloak Configuration - -Now that Keycloak is running, we also need to add a new `groupware` client to the Keycloak `OpenCloud` realm in order to be able to use our command-line scripts and other test components. - -To do so, use your preferred web browser and -* head over to -* authenticate as `admin` with password `admin` (those credentials are defined in the `.env` file mentioned above, see `KEYCLOAK_ADMIN_USER` and `KEYCLOAK_ADMIN_PASSWORD`) -* select the `OpenCloud` realm in the drop-down list in the top left corner (the realm is defined in the `.env` file, see `KEYCLOAK_REALM`) -* then select the "Clients" menu item on the left -* in the "Clients list" tab, push the "Create client" button: - * Client type: `OpenID Connect` - * Client ID: `groupware` -* click the "Next" button: - * Client authentication: Off - * Authorization: Off - * Authentication flow: make sure "Direct access grants" is checked -* click the "Next" button and leave the fields there empty to stick to the defaults -* click "Save" - -To check whether it works correctly: -```bash -curl -ks -D- -X POST "https://keycloak.opencloud.test/realms/openCloud/protocol/openid-connect/token" -d username=alan -d password=demo -d grant_type=password -d client_id=groupware -d scope=openid -``` -should provide you with a JSON response that contains an `access_token`. - -If it is not set up correctly, it should give you this instead: -```json -{"error":"invalid_client","error_description":"Invalid client or Invalid client credentials"} -``` - -## Checking +## Checking Services To check whether the various services are running correctly: @@ -195,6 +178,19 @@ dn: uid=margaret,ou=users,dc=opencloud,dc=eu ``` +### Keycloak + +To check whether it works correctly: +```bash +curl -ks -D- -X POST "https://keycloak.opencloud.test/realms/openCloud/protocol/openid-connect/token" -d username=alan -d password=demo -d grant_type=password -d client_id=groupware -d scope=openid +``` +should provide you with a JSON response that contains an `access_token`. + +If it is not set up correctly, it should give you this instead: +```json +{"error":"invalid_client","error_description":"Invalid client or Invalid client credentials"} +``` + ### Stalwart To then test the IMAP authentication with Stalwart, run the following command on your host (requires the `openssl` CLI tool): @@ -218,20 +214,12 @@ to which one should receive the following response: A OK [CAPABILITY IMAP4rev2 ...] Authentication successful ``` -### Keycloak - -As mentioned previously, use the following command on your host to retrieve an access token from Keycloak: - -```bash -curl -ks -D- -X POST "https://keycloak.opencloud.test/realms/openCloud/protocol/openid-connect/token" -d username=alan -d password=demo -d grant_type=password -d client_id=groupware -d scope=openid -``` - ## Feeding an Inbox Once a [Stalwart](https://stalw.art/) container is running (using the Docker Compose setup as explained above), use [`imap-filler`](https://github.com/opencloud-eu/imap-filler/) to populate the inbox folder via IMAP APPEND: ```bash -cd ~/src/opencloud/ +cd "$OCDIR/" git clone git@github.com:opencloud-eu/imap-filler.git cd ./imap-filler go run . --empty=true --username=alan --password=demo \ @@ -242,7 +230,6 @@ go run . --empty=true --username=alan --password=demo \ If you run the `opencloud` service as a container, use the following script to update the container image and restart it: - ```bash oc-full-update ``` @@ -253,16 +240,14 @@ If you run it from your IDE, there is obviously no need to do that. The REST API documentation is extracted from the source code structure and documentation using [`go-swagger`](https://goswagger.io/go-swagger/), which needs to be installed locally as a prerequisite: - ```bash go install github.com/go-swagger/go-swagger/cmd/swagger@latest ``` The build chain is integrated within the `Makefile` in `services/groupware/`: - ```bash -cd services/groupware/ +cd "$OCDIR/opencloud/services/groupware/" make apidoc-static ``` @@ -333,7 +318,7 @@ authentication.fallback-admin.user = "mailadmin" To start with a Stalwart container from scratch, removing all the data (including emails): ```bash -cd deployments/examples/opencloud_full +cd "$OCDIR/opencloud/deployments/examples/opencloud_full" docker compose stop stalwart docker compose rm stalwart docker volume rm opencloud_full_stalwart-data opencloud_full_stalwart-logs