Add Keycloak CR support for Tracing options (#35703)

Closes #32092

Signed-off-by: Martin Bartoš <mabartos@redhat.com>
This commit is contained in:
Martin Bartoš
2024-12-12 12:59:27 +01:00
committed by GitHub
parent ad679b8729
commit 41356dff24
11 changed files with 370 additions and 18 deletions

View File

@@ -125,6 +125,7 @@ It means the `opentelemetry` feature is enabled by default.
There were made multiple improvements to the tracing capabilities in {project_name} such as:
* *Configuration via Keycloak CR* in {project_name} Operator
* *Custom spans* for:
** Incoming/outgoing HTTP requests including Identity Providers brokerage
** Database operations and connections

View File

@@ -163,8 +163,11 @@ For more information, see the https://www.w3.org/TR/trace-context/#security-cons
== Tracing in Kubernetes environment
When the tracing is enabled when using the {project_name} Operator, certain information about the deployment is propagated to the underlying containers.
NOTE: There is no support for tracing configuration in {project_name} CR yet, so the `additionalOptions` can be used to the `tracing-enabled` property and other tracing options.
=== Configuration via Keycloak CR
You can change tracing configuration via Keycloak CR. For more information, see the <@links.operator id="advanced-configuration" anchor="_tracing_opentelemetry" />.
=== Filter traces based on Kubernetes attributes
You can filter out the required traces in your tracing backend based on their tags:
* `service.name` - {project_name} deployment name
@@ -173,4 +176,6 @@ You can filter out the required traces in your tracing backend based on their ta
{project_name} Operator automatically sets the `KC_TRACING_SERVICE_NAME` and `KC_TRACING_RESOURCE_ATTRIBUTES` environment variables for each {project_name} container included in pods it manages.
NOTE: The `KC_TRACING_RESOURCE_ATTRIBUTES` variable always contains (if not overridden) the `k8s.namespace.name` attribute representing current namespace.
</@tmpl.guide>

View File

@@ -271,6 +271,8 @@ NOTE: If you are using a custom image, the Operator is *unaware* of any configur
For instance, it may cause that the management interface uses the `https` schema, but the Operator accesses it via `http` when the TLS settings is specified in the custom image.
To ensure proper TLS configuration, use the `tlsSecret` and `truststores` fields in the Keycloak CR so that the Operator can reflect that.
For more details, see <@links.server id="management-interface" />.
=== Truststores
If you need to provide trusted certificates, the Keycloak CR provides a top level feature for configuring the server's truststore as discussed in <@links.server id="keycloak-truststore"/>.
@@ -316,6 +318,37 @@ If a master realm has already been created for you cluster, then the spec.boostr
For more information on how to bootstrap a temporary admin user or service account and recover lost admin access, refer to the <@links.server id="bootstrap-admin-recovery"/> guide.
=== Tracing (OpenTelemetry)
Tracing allows for detailed monitoring of each request's lifecycle, which helps quickly identify and diagnose issues, leading to more efficient debugging and maintenance.
You can change tracing configuration via Keycloak CR fields as follows:
[source,yaml]
----
apiVersion: k8s.keycloak.org/v2alpha1
kind: Keycloak
metadata:
name: example-kc
spec:
tracing:
enabled: true # default 'false'
endpoint: http://my-tracing:4317 # default 'http://localhost:4317'
samplerType: parentbased_traceidratio # default 'traceidratio'
samplerRatio: 0.01 # default '1'
resourceAttributes:
some.attribute: something
additionalOptions:
- name: tracing-jdbc-enabled
value: false # default 'true'
----
These fields should reflect 1:1 association with `tracing-*` options that contain more information.
NOTE: The `tracing-jdbc-enabled` is not promoted as a first-class citizen as it might not be well managed in the future, so it needs to be set via the `additionalOptions` field.
For more details about tracing, see <@links.observability id="tracing" />.
=== Network Policies (Experimental)
NetworkPolicies allow you to specify rules for traffic flow within your cluster, and also between Pods and the outside world.

View File

@@ -71,6 +71,7 @@ import java.util.stream.Stream;
import static org.keycloak.operator.Utils.addResources;
import static org.keycloak.operator.controllers.KeycloakDistConfigurator.getKeycloakOptionEnvVarName;
import static org.keycloak.operator.crds.v2alpha1.CRDUtils.isTlsConfigured;
import static org.keycloak.operator.crds.v2alpha1.deployment.spec.TracingSpec.convertTracingAttributesToString;
public class KeycloakDeploymentDependentResource extends CRUDKubernetesDependentResource<StatefulSet, Keycloak> {
@@ -421,20 +422,7 @@ public class KeycloakDeploymentDependentResource extends CRUDKubernetesDependent
// include the kube CA if the user is not controlling KC_TRUSTSTORE_PATHS via the unsupported or the additional
varMap.putIfAbsent(KC_TRUSTSTORE_PATHS, new EnvVarBuilder().withName(KC_TRUSTSTORE_PATHS).withValue(truststores).build());
varMap.putIfAbsent(KC_TRACING_SERVICE_NAME,
new EnvVarBuilder().withName(KC_TRACING_SERVICE_NAME)
.withValue(keycloakCR.getMetadata().getName())
.build()
);
// Possible OTel k8s attributes convention can be found here: https://opentelemetry.io/docs/specs/semconv/attributes-registry/k8s/#kubernetes-attributes
var tracingAttributes = Map.of("k8s.namespace.name", keycloakCR.getMetadata().getNamespace());
varMap.putIfAbsent(KC_TRACING_RESOURCE_ATTRIBUTES,
new EnvVarBuilder().withName(KC_TRACING_RESOURCE_ATTRIBUTES)
.withValue(tracingAttributes.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(",")))
.build()
);
setTracingEnvVars(keycloakCR, varMap);
var envVars = new ArrayList<>(varMap.values());
baseDeployment.getSpec().getTemplate().getSpec().getContainers().get(0).setEnv(envVars);
@@ -448,6 +436,37 @@ public class KeycloakDeploymentDependentResource extends CRUDKubernetesDependent
allSecrets.addAll(serverConfigSecretsNames);
}
private static void setTracingEnvVars(Keycloak keycloakCR, Map<String, EnvVar> varMap) {
varMap.putIfAbsent(KC_TRACING_SERVICE_NAME,
new EnvVarBuilder().withName(KC_TRACING_SERVICE_NAME)
.withValue(keycloakCR.getMetadata().getName())
.build()
);
// Possible OTel k8s attributes convention can be found here: https://opentelemetry.io/docs/specs/semconv/attributes-registry/k8s/#kubernetes-attributes
var tracingAttributes = Map.of("k8s.namespace.name", keycloakCR.getMetadata().getNamespace());
if (varMap.containsKey(KC_TRACING_RESOURCE_ATTRIBUTES)) {
// append 'tracingAttributes' to the existing attributes defined in the 'KC_TRACING_RESOURCE_ATTRIBUTES' env var
var existingAttributes = convertTracingAttributesToMap(varMap);
tracingAttributes.forEach(existingAttributes::putIfAbsent);
varMap.get(KC_TRACING_RESOURCE_ATTRIBUTES).setValue(convertTracingAttributesToString(existingAttributes));
} else {
varMap.put(KC_TRACING_RESOURCE_ATTRIBUTES,
new EnvVarBuilder().withName(KC_TRACING_RESOURCE_ATTRIBUTES)
.withValue(convertTracingAttributesToString(tracingAttributes))
.build()
);
}
}
private static Map<String, String> convertTracingAttributesToMap(Map<String, EnvVar> envVars) {
return Arrays.stream(Optional.ofNullable(envVars.get(KC_TRACING_RESOURCE_ATTRIBUTES).getValue()).orElse("").split(","))
.filter(entry -> entry.contains("="))
.map(entry -> entry.split("=", 2))
.collect(Collectors.toMap(entry -> entry[0], entry -> entry[1]));
}
private List<EnvVar> getDefaultAndAdditionalEnvVars(Keycloak keycloakCR) {
// default config values
List<ValueOrSecret> serverConfigsList = new ArrayList<>(Constants.DEFAULT_DIST_CONFIG_LIST);

View File

@@ -22,7 +22,7 @@ import io.fabric8.kubernetes.api.model.EnvVarBuilder;
import io.fabric8.kubernetes.api.model.EnvVarSourceBuilder;
import io.fabric8.kubernetes.api.model.SecretKeySelector;
import io.quarkus.logging.Log;
import jakarta.enterprise.context.ApplicationScoped;
import org.keycloak.common.util.CollectionUtil;
import org.keycloak.operator.Constants;
import org.keycloak.operator.crds.v2alpha1.deployment.Keycloak;
@@ -35,6 +35,7 @@ import org.keycloak.operator.crds.v2alpha1.deployment.spec.HostnameSpec;
import org.keycloak.operator.crds.v2alpha1.deployment.spec.HttpManagementSpec;
import org.keycloak.operator.crds.v2alpha1.deployment.spec.HttpSpec;
import org.keycloak.operator.crds.v2alpha1.deployment.spec.ProxySpec;
import org.keycloak.operator.crds.v2alpha1.deployment.spec.TracingSpec;
import org.keycloak.operator.crds.v2alpha1.deployment.spec.TransactionsSpec;
import java.util.ArrayList;
@@ -48,8 +49,6 @@ import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
import jakarta.enterprise.context.ApplicationScoped;
import static io.smallrye.config.common.utils.StringUtil.replaceNonAlphanumericByUnderscores;
/**
@@ -68,6 +67,7 @@ public class KeycloakDistConfigurator {
// register the configuration mappers for the various parts of the keycloak cr
configureHostname();
configureFeatures();
configureTracing();
configureTransactions();
configureHttp();
configureDatabase();
@@ -124,6 +124,18 @@ public class KeycloakDistConfigurator {
.mapOptionFromCollection("features-disabled", FeatureSpec::getDisabledFeatures);
}
void configureTracing() {
optionMapper(keycloakCR -> keycloakCR.getSpec().getTracingSpec())
.mapOption("tracing-enabled", TracingSpec::getEnabled)
.mapOption("tracing-service-name", TracingSpec::getServiceName)
.mapOption("tracing-endpoint", TracingSpec::getEndpoint)
.mapOption("tracing-protocol", TracingSpec::getProtocol)
.mapOption("tracing-sampler-type", TracingSpec::getSamplerType)
.mapOption("tracing-sampler-ratio", TracingSpec::getSamplerRatio)
.mapOption("tracing-compression", TracingSpec::getCompression)
.mapOption("tracing-resource-attributes", TracingSpec::getResourceAttributesString);
}
void configureTransactions() {
optionMapper(keycloakCR -> keycloakCR.getSpec().getTransactionsSpec())
.mapOption("transaction-xa-enabled", TransactionsSpec::isXaEnabled);

View File

@@ -31,6 +31,7 @@ import org.keycloak.operator.crds.v2alpha1.deployment.spec.IngressSpec;
import org.keycloak.operator.crds.v2alpha1.deployment.spec.NetworkPolicySpec;
import org.keycloak.operator.crds.v2alpha1.deployment.spec.ProxySpec;
import org.keycloak.operator.crds.v2alpha1.deployment.spec.SchedulingSpec;
import org.keycloak.operator.crds.v2alpha1.deployment.spec.TracingSpec;
import org.keycloak.operator.crds.v2alpha1.deployment.spec.TransactionsSpec;
import org.keycloak.operator.crds.v2alpha1.deployment.spec.Truststore;
import org.keycloak.operator.crds.v2alpha1.deployment.spec.UnsupportedSpec;
@@ -125,6 +126,10 @@ public class KeycloakSpec {
@JsonPropertyDescription("Controls the ingress traffic flow into Keycloak pods.")
private NetworkPolicySpec networkPolicySpec;
@JsonProperty("tracing")
@JsonPropertyDescription("In this section you can configure OpenTelemetry Tracing for Keycloak.")
private TracingSpec tracingSpec;
public HttpSpec getHttpSpec() {
return httpSpec;
}
@@ -290,4 +295,12 @@ public class KeycloakSpec {
public void setNetworkPolicySpec(NetworkPolicySpec networkPolicySpec) {
this.networkPolicySpec = networkPolicySpec;
}
public TracingSpec getTracingSpec() {
return tracingSpec;
}
public void setTracingSpec(TracingSpec tracingSpec) {
this.tracingSpec = tracingSpec;
}
}

View File

@@ -0,0 +1,138 @@
/*
* Copyright 2024 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.operator.crds.v2alpha1.deployment.spec;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonPropertyDescription;
import io.sundr.builder.annotations.Buildable;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.stream.Collectors;
@JsonInclude(JsonInclude.Include.NON_NULL)
@Buildable(editableEnabled = false, builderPackage = "io.fabric8.kubernetes.api.builder")
public class TracingSpec {
@JsonPropertyDescription("Enables the OpenTelemetry tracing.")
private Boolean enabled;
@JsonPropertyDescription("OpenTelemetry endpoint to connect to.")
private String endpoint;
@JsonPropertyDescription("OpenTelemetry service name. Takes precedence over 'service.name' defined in the 'resourceAttributes' map.")
private String serviceName;
@JsonPropertyDescription("OpenTelemetry protocol used for the telemetry data (default 'grpc'). For more information, check the Tracing guide.")
private String protocol;
@JsonPropertyDescription("OpenTelemetry sampler to use for tracing (default 'traceidratio'). For more information, check the Tracing guide.")
private String samplerType;
@JsonPropertyDescription("OpenTelemetry sampler ratio. Probability that a span will be sampled. Expected double value in interval <0,1).")
private Double samplerRatio;
@JsonPropertyDescription("OpenTelemetry compression method used to compress payloads. If unset, compression is disabled. Possible values are: gzip, none.")
private String compression;
@JsonPropertyDescription("OpenTelemetry resource attributes present in the exported trace to characterize the telemetry producer.")
private Map<String, String> resourceAttributes;
public Boolean getEnabled() {
return enabled;
}
public void setEnabled(Boolean enabled) {
this.enabled = enabled;
}
public String getEndpoint() {
return endpoint;
}
public void setEndpoint(String endpoint) {
this.endpoint = endpoint;
}
public String getServiceName() {
return serviceName;
}
public void setServiceName(String serviceName) {
this.serviceName = serviceName;
}
public String getProtocol() {
return protocol;
}
public void setProtocol(String protocol) {
this.protocol = protocol;
}
public String getSamplerType() {
return samplerType;
}
public void setSamplerType(String samplerType) {
this.samplerType = samplerType;
}
public Double getSamplerRatio() {
return samplerRatio;
}
public void setSamplerRatio(Double samplerRatio) {
this.samplerRatio = samplerRatio;
}
public String getCompression() {
return compression;
}
public void setCompression(String compression) {
this.compression = compression;
}
public Map<String, String> getResourceAttributes() {
if (resourceAttributes == null) {
resourceAttributes = new LinkedHashMap<>();
}
return resourceAttributes;
}
// resource attributes in format key=val delimited by comma
@JsonIgnore
public String getResourceAttributesString() {
return convertTracingAttributesToString(getResourceAttributes());
}
public void setResourceAttributes(Map<String, String> resourceAttributes) {
this.resourceAttributes = resourceAttributes;
}
/**
* Convert resource attributes in format key=val delimited by comma to string
*/
public static String convertTracingAttributesToString(Map<String, String> attributes) {
return attributes.entrySet().stream()
.map(attr -> String.format("%s=%s", attr.getKey(), attr.getValue()))
.collect(Collectors.joining(","));
}
}

View File

@@ -17,6 +17,7 @@
package org.keycloak.operator.testsuite.integration;
import io.fabric8.kubernetes.api.model.EnvVar;
import io.fabric8.kubernetes.api.model.EnvVarBuilder;
import io.fabric8.kubernetes.api.model.LocalObjectReference;
import io.fabric8.kubernetes.api.model.LocalObjectReferenceBuilder;
@@ -48,6 +49,7 @@ import org.keycloak.operator.crds.v2alpha1.deployment.ValueOrSecret;
import org.keycloak.operator.crds.v2alpha1.deployment.spec.BootstrapAdminSpec;
import org.keycloak.operator.crds.v2alpha1.deployment.spec.FeatureSpec;
import org.keycloak.operator.crds.v2alpha1.deployment.spec.HostnameSpecBuilder;
import org.keycloak.operator.crds.v2alpha1.deployment.spec.TracingSpecBuilder;
import org.keycloak.operator.testsuite.unit.WatchedResourcesTest;
import org.keycloak.operator.testsuite.utils.CRAssert;
import org.keycloak.operator.testsuite.utils.K8sUtils;
@@ -58,6 +60,7 @@ import java.util.Base64;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
@@ -770,6 +773,82 @@ public class KeycloakDeploymentTest extends BaseOperatorTest {
assertThat(limits.get("memory")).isEqualTo(config.keycloak().resources().limits().memory());
}
@Test
public void testTracingSpec() {
var kc = getTestKeycloakDeployment(false);
kc.getSpec().setStartOptimized(false);
var tracingSpec = new TracingSpecBuilder()
.withEnabled()
.withEndpoint("http://0.0.0.0:4317")
.withServiceName("my-best-keycloak")
.withProtocol("http/protobuf")
.withSamplerType("parentbased_traceidratio")
.withSamplerRatio(0.01)
.withCompression("gzip")
.withResourceAttributes(Map.of(
"something.a", "keycloak-rocks",
"something.b", "keycloak-rocks2"))
.build();
kc.getSpec().setTracingSpec(tracingSpec);
deployKeycloak(k8sclient, kc, true);
var pods = k8sclient
.pods()
.inNamespace(namespace)
.withLabels(Constants.DEFAULT_LABELS)
.list()
.getItems();
assertThat(pods).isNotNull();
assertThat(pods).isNotEmpty();
var map = pods.get(0).getSpec().getContainers().get(0).getEnv().stream()
.filter(Objects::nonNull).filter(f -> f.getName().startsWith("KC_TRACING_"))
.collect(Collectors.toMap(EnvVar::getName, EnvVar::getValue));
assertThat(map).isNotNull();
assertThat(map).isNotEmpty();
// assertions
var enabled = map.get("KC_TRACING_ENABLED");
assertThat(enabled).isNotNull();
assertThat(enabled).isEqualTo("true");
var endpoint = map.get("KC_TRACING_ENDPOINT");
assertThat(endpoint).isNotNull();
assertThat(endpoint).isEqualTo("http://0.0.0.0:4317");
var serviceName = map.get("KC_TRACING_SERVICE_NAME");
assertThat(serviceName).isNotNull();
assertThat(serviceName).isEqualTo("my-best-keycloak");
var protocol = map.get("KC_TRACING_PROTOCOL");
assertThat(protocol).isNotNull();
assertThat(protocol).isEqualTo("http/protobuf");
var samplerType = map.get("KC_TRACING_SAMPLER_TYPE");
assertThat(samplerType).isNotNull();
assertThat(samplerType).isEqualTo("parentbased_traceidratio");
var samplerRatio = map.get("KC_TRACING_SAMPLER_RATIO");
assertThat(samplerRatio).isNotNull();
assertThat(samplerRatio).isEqualTo("0.01");
var compression = map.get("KC_TRACING_COMPRESSION");
assertThat(compression).isNotNull();
assertThat(compression).isEqualTo("gzip");
var resourceAttributes = map.get("KC_TRACING_RESOURCE_ATTRIBUTES");
assertThat(resourceAttributes).isNotNull();
assertThat(resourceAttributes).contains("something.a=keycloak-rocks");
assertThat(resourceAttributes).contains("something.b=keycloak-rocks2");
assertThat(resourceAttributes).contains(String.format("k8s.namespace.name=%s", namespace));
}
private void handleFakeImagePullSecretCreation(Keycloak keycloakCR,
String secretDescriptorFilename) {

View File

@@ -27,6 +27,7 @@ import org.keycloak.operator.crds.v2alpha1.deployment.spec.DatabaseSpec;
import org.keycloak.operator.crds.v2alpha1.deployment.spec.FeatureSpec;
import org.keycloak.operator.crds.v2alpha1.deployment.spec.HostnameSpec;
import org.keycloak.operator.crds.v2alpha1.deployment.spec.HttpManagementSpec;
import org.keycloak.operator.crds.v2alpha1.deployment.spec.TracingSpec;
import org.keycloak.operator.crds.v2alpha1.deployment.spec.TransactionsSpec;
import org.keycloak.operator.crds.v2alpha1.realmimport.KeycloakRealmImport;
import org.keycloak.operator.testsuite.utils.K8sUtils;
@@ -37,6 +38,7 @@ import java.util.Map;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.emptyString;
import static org.hamcrest.Matchers.hasEntry;
import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.hasProperty;
import static org.hamcrest.Matchers.is;
@@ -175,6 +177,29 @@ public class CRSerializationTest {
assertThat(limitMemQuantity.getFormat(), is("M"));
}
@Test
public void tracingSpecification() {
Keycloak keycloak = Serialization.unmarshal(this.getClass().getResourceAsStream("/test-serialization-keycloak-cr.yml"), Keycloak.class);
TracingSpec tracing = keycloak.getSpec().getTracingSpec();
assertThat(tracing, notNullValue());
assertThat(tracing.getEnabled(), is(true));
assertThat(tracing.getEndpoint(), is("http://my-tracing:4317"));
assertThat(tracing.getServiceName(), is("my-best-keycloak"));
assertThat(tracing.getProtocol(), is("http/protobuf"));
assertThat(tracing.getSamplerType(), is("parentbased_traceidratio"));
assertThat(tracing.getSamplerRatio(), is(0.01));
assertThat(tracing.getCompression(), is("gzip"));
var attributes = tracing.getResourceAttributes();
assertThat(attributes, notNullValue());
assertThat(attributes.size(), is(2));
assertThat(attributes, hasEntry("service.namespace", "keycloak-namespace"));
assertThat(attributes, hasEntry("service.name", "custom-service-name"));
}
@Test
public void resourcesSpecificationOnlyLimit() {
final Keycloak keycloak = K8sUtils.getResourceFromFile("test-serialization-keycloak-cr-with-empty-list.yml", Keycloak.class);

View File

@@ -167,6 +167,22 @@ public class KeycloakDistConfiguratorTest {
testFirstClassCitizen(expectedValues);
}
@Test
public void tracing() {
final Map<String, String> expectedValues = Map.of(
"tracing-enabled", "true",
"tracing-endpoint", "http://my-tracing:4317",
"tracing-service-name", "my-best-keycloak",
"tracing-protocol", "http/protobuf",
"tracing-sampler-type", "parentbased_traceidratio",
"tracing-sampler-ratio", "0.01",
"tracing-compression", "gzip",
"tracing-resource-attributes", "service.namespace=keycloak-namespace,service.name=custom-service-name"
);
testFirstClassCitizen(expectedValues);
}
/* UTILS */
private void testFirstClassCitizen(Map<String, String> expectedValues) {

View File

@@ -59,6 +59,17 @@ spec:
- step-up-authentication
transaction:
xaEnabled: false
tracing:
enabled: true
endpoint: http://my-tracing:4317
serviceName: my-best-keycloak
protocol: http/protobuf
samplerType: parentbased_traceidratio
samplerRatio: 0.01
compression: gzip
resourceAttributes:
service.namespace: keycloak-namespace
service.name: custom-service-name
resources:
requests:
cpu: "500m"