[Keycloak Test Framework] Infinispan cache + ClusterlessTestSuite configuration (#42172)

* [Keycloak Test Framework] Infinispan server + ClusterlessTestSuite and MultisiteTestSuite configuration

Signed-off-by: Lukas Hanusovsky <lhanusov@redhat.com>

* Utilise ClientIntelligence.BASIC to ensure that internal docker IPs
never used by Infinispan client

Signed-off-by: Ryan Emerson <remerson@ibm.com>

* Code refactoring + properties utility

Signed-off-by: Lukas Hanusovsky <lhanusov@redhat.com>

---------

Signed-off-by: Lukas Hanusovsky <lhanusov@redhat.com>
Signed-off-by: Ryan Emerson <remerson@ibm.com>
Co-authored-by: Ryan Emerson <remerson@ibm.com>

test-framework/LOGGING.md

@@ -42,7 +42,8 @@ Some useful categories include:
 * `org.keycloak.testframework` - Logging from the test framework itself. Setting this to `DEBUG` can be helpful to debug any issues with the test framework itself, or custom suppliers.
 * `org.keycloak` - Logging from the Keycloak server. If you set this to `DEBUG` for example, but don't want debug from the test framework, also explicitly set `org.keycloak.testframework` to for example `INFO`
 * `managed.keycloak` - Log output from the managed Keycloak server if you are running the server in `distribution` mode (which is the default)
-* `managed.db` - Output from database containers are included in this category. Standard out is logged with `DEBUG` level, while standard error is logged with `WARN` level  
+* `managed.db` - Output from database containers are included in this category. Standard out is logged with `DEBUG` level, while standard error is logged with `WARN` level
+* `managed.infinispan` - Output from the external Infinispan container is included in this category. Standard out is logged with `DEBUG` level, while standard error is logged with `WARN` level
 
 ### Enable log filtering
 
@@ -70,6 +71,7 @@ kc.test.log.category."org.keycloak.testframework".level=INFO
 kc.test.log.category."org.keycloak".level=WARN
 kc.test.log.category."managed.keycloak".level=WARN
 kc.test.log.category."managed.db".level=WARN
+kc.test.log.category."managed.infinispan".level=WARN
 ```
 
 This should serve as a good starting point balancing the need of log information with not producing too much noise.
@@ -89,6 +91,7 @@ KC_TEST_LOG_CATEGORY__ORG_KEYCLOAK___LEVEL=DEBUG
 KC_TEST_LOG_CATEGORY__ORG_KEYCLOAK_TEST__LEVEL=DEBUG
 KC_TEST_LOG_CATEGORY__MANAGED_KEYCLOAK__LEVEL=DEBUG
 KC_TEST_LOG_CATEGORY__MANAGED_DB__LEVEL=DEBUG
+KC_TEST_LOG_CATEGORY__MANAGED_INFINISPAN__LEVEL=DEBUG
 ```
 
 ### Examples using environment variables

test-framework/clustering/src/main/java/org/keycloak/testframework/server/ClusteredKeycloakServer.java

@@ -26,6 +26,7 @@ import org.infinispan.server.test.core.CountdownLatchLoggingConsumer;
 import org.jboss.logging.Logger;
 import org.keycloak.it.utils.DockerKeycloakDistribution;
 import org.keycloak.testframework.clustering.LoadBalancer;
+import org.keycloak.testframework.infinispan.CacheType;
 import org.keycloak.testframework.logging.JBossLogConsumer;
 import org.testcontainers.images.RemoteDockerImage;
 import org.testcontainers.utility.DockerImageName;
@@ -56,6 +57,10 @@ public class ClusteredKeycloakServer implements KeycloakServer {
         int numServers = containers.length;
         CountdownLatchLoggingConsumer clusterLatch = new CountdownLatchLoggingConsumer(numServers, String.format(CLUSTER_VIEW_REGEX, numServers));
         String[] imagePeServer = null;
+
+        // Infinispan clustered cache
+        configBuilder.cache(CacheType.ISPN);
+
         if (images == null || images.isEmpty() || (imagePeServer = images.split(",")).length == 1) {
             startContainersWithSameImage(configBuilder, imagePeServer == null ? SNAPSHOT_IMAGE : imagePeServer[0], clusterLatch);
         } else {

test-framework/clustering/src/main/java/org/keycloak/testframework/server/ClusteredKeycloakServerSupplier.java

@@ -32,9 +32,4 @@ public class ClusteredKeycloakServerSupplier extends AbstractKeycloakServerSuppl
     public Logger getLogger() {
         return LOGGER;
     }
-
-    @Override
-    protected String cache() {
-        return "ispn";
-    }
 }

test-framework/core/src/main/java/org/keycloak/testframework/CoreTestFrameworkExtension.java

@@ -2,6 +2,7 @@ package org.keycloak.testframework;
 
 import org.keycloak.testframework.admin.AdminClientFactorySupplier;
 import org.keycloak.testframework.admin.AdminClientSupplier;
+import org.keycloak.testframework.infinispan.InfinispanExternalServerSupplier;
 import org.keycloak.testframework.database.DevFileDatabaseSupplier;
 import org.keycloak.testframework.database.DevMemDatabaseSupplier;
 import org.keycloak.testframework.database.TestDatabase;
@@ -43,7 +44,8 @@ public class CoreTestFrameworkExtension implements TestFrameworkExtension {
                 new EventsSupplier(),
                 new AdminEventsSupplier(),
                 new HttpClientSupplier(),
-                new HttpServerSupplier()
+                new HttpServerSupplier(),
+                new InfinispanExternalServerSupplier()
         );
     }
 

test-framework/core/src/main/java/org/keycloak/testframework/annotations/InjectInfinispanServer.java

@@ -0,0 +1,15 @@
+package org.keycloak.testframework.annotations;
+
+import org.keycloak.testframework.injection.LifeCycle;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.TYPE)
+public @interface InjectInfinispanServer {
+
+    LifeCycle lifecycle() default LifeCycle.GLOBAL;
+}

test-framework/core/src/main/java/org/keycloak/testframework/database/DatabaseProperties.java

@@ -1,24 +0,0 @@
-package org.keycloak.testframework.database;
-
-import java.io.IOException;
-import java.util.Properties;
-
-public class DatabaseProperties {
-
-    private static final Properties PROPERTIES = loadProperties();
-
-    public static String getContainerImageName(String database) {
-        return PROPERTIES.getProperty(database + ".container");
-    }
-
-    private static Properties loadProperties() {
-        Properties properties = new Properties();
-        try {
-            properties.load(DatabaseProperties.class.getResourceAsStream("database.properties"));
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-        return properties;
-    }
-
-}

test-framework/core/src/main/java/org/keycloak/testframework/infinispan/CacheType.java

@@ -0,0 +1,10 @@
+package org.keycloak.testframework.infinispan;
+
+public enum CacheType {
+
+    // Local Infinispan Cache for Embedded Deployment only
+    LOCAL,
+
+    // Clustered Infinispan Cache can be set for Embedded or External Deployment
+    ISPN;
+}

test-framework/core/src/main/java/org/keycloak/testframework/infinispan/InfinispanExternalServer.java

@@ -0,0 +1,42 @@
+package org.keycloak.testframework.infinispan;
+
+import java.util.Map;
+
+import org.infinispan.server.test.core.InfinispanContainer;
+import org.jboss.logging.Logger;
+import org.keycloak.testframework.logging.JBossLogConsumer;
+import org.keycloak.testframework.util.ContainerImages;
+
+public class InfinispanExternalServer extends InfinispanContainer implements InfinispanServer {
+
+    private static final String USER = "keycloak";
+    private static final String PASSWORD = "Password1!";
+    private static final String HOST = "127.0.0.1";
+
+    public static InfinispanExternalServer create() {
+        return new InfinispanExternalServer(ContainerImages.getContainerImageName("infinispan"));
+    }
+
+    @SuppressWarnings("resource")
+    private InfinispanExternalServer(String dockerImageName) {
+        super(dockerImageName);
+        withUser(USER);
+        withPassword(PASSWORD);
+        withLogConsumer(new JBossLogConsumer(Logger.getLogger("managed.infinispan")));
+        addFixedExposedPort(11222, 11222);
+    }
+
+    @Override
+    public Map<String, String> serverConfig() {
+        return Map.of(
+                "cache-remote-host", HOST,
+                "cache-remote-username", USER,
+                "cache-remote-password", PASSWORD,
+                "cache-remote-tls-enabled", "false",
+                "spi-cache-embedded-default-site-name", "ispn",
+                "spi-load-balancer-check-remote-poll-interval", "500",
+                "spi-cache-remote-default-client-intelligence", "BASIC",
+                "-Dkc.cache-remote-create-caches", "true"
+        );
+    }
+}

test-framework/core/src/main/java/org/keycloak/testframework/infinispan/InfinispanExternalServerSupplier.java

@@ -0,0 +1,54 @@
+package org.keycloak.testframework.infinispan;
+
+import org.jboss.logging.Logger;
+import org.keycloak.testframework.annotations.InjectInfinispanServer;
+import org.keycloak.testframework.injection.InstanceContext;
+import org.keycloak.testframework.injection.RequestedInstance;
+import org.keycloak.testframework.injection.Supplier;
+import org.keycloak.testframework.injection.SupplierOrder;
+import org.keycloak.testframework.server.KeycloakServerConfigBuilder;
+import org.keycloak.testframework.server.KeycloakServerConfigInterceptor;
+
+public class InfinispanExternalServerSupplier implements Supplier<InfinispanServer, InjectInfinispanServer>, KeycloakServerConfigInterceptor<InfinispanServer, InjectInfinispanServer> {
+
+    private static final Logger LOGGER = Logger.getLogger(InfinispanExternalServerSupplier.class);
+
+    @Override
+    public InfinispanServer getValue(InstanceContext<InfinispanServer, InjectInfinispanServer> instanceContext) {
+        InfinispanServer server = InfinispanExternalServer.create();
+        getLogger().info("Starting Infinispan Server");
+
+        long start = System.currentTimeMillis();
+
+        server.start();
+
+        getLogger().infov("Infinispan server started in {0} ms", System.currentTimeMillis() - start);
+        return server;
+    }
+
+    @Override
+    public void close(InstanceContext<InfinispanServer, InjectInfinispanServer> instanceContext) {
+        instanceContext.getValue().stop();
+    }
+
+    @Override
+    public boolean compatible(InstanceContext<InfinispanServer, InjectInfinispanServer> a, RequestedInstance<InfinispanServer, InjectInfinispanServer> b) {
+        return a.getSupplier().getRef(a.getAnnotation()).equals(b.getSupplier().getRef(a.getAnnotation()));
+    }
+
+    @Override
+    public int order() {
+        return SupplierOrder.BEFORE_KEYCLOAK_SERVER;
+    }
+
+    @Override
+    public KeycloakServerConfigBuilder intercept(KeycloakServerConfigBuilder config, InstanceContext<InfinispanServer, InjectInfinispanServer> instanceContext) {
+        InfinispanServer ispnServer = instanceContext.getValue();
+
+        return config.options(ispnServer.serverConfig());
+    }
+
+    public Logger getLogger() {
+        return LOGGER;
+    }
+}

test-framework/core/src/main/java/org/keycloak/testframework/infinispan/InfinispanServer.java

@@ -0,0 +1,12 @@
+package org.keycloak.testframework.infinispan;
+
+import java.util.Map;
+
+public interface InfinispanServer {
+
+    void start();
+
+    void stop();
+
+    Map<String, String> serverConfig();
+}

test-framework/core/src/main/java/org/keycloak/testframework/server/AbstractKeycloakServerSupplier.java

@@ -2,6 +2,7 @@ package org.keycloak.testframework.server;
 
 import org.jboss.logging.Logger;
 import org.keycloak.testframework.annotations.KeycloakIntegrationTest;
+import org.keycloak.testframework.infinispan.InfinispanServer;
 import org.keycloak.testframework.config.Config;
 import org.keycloak.testframework.database.TestDatabase;
 import org.keycloak.testframework.injection.AbstractInterceptorHelper;
@@ -21,7 +22,6 @@ public abstract class AbstractKeycloakServerSupplier implements Supplier<Keycloa
         KeycloakServerConfig serverConfig = SupplierHelpers.getInstance(annotation.config());
 
         KeycloakServerConfigBuilder command = KeycloakServerConfigBuilder.startDev()
-                .cache(cache())
                 .bootstrapAdminClient(Config.getAdminClientId(), Config.getAdminClientSecret())
                 .bootstrapAdminUser(Config.getAdminUsername(), Config.getAdminPassword());
 
@@ -35,10 +35,16 @@ public abstract class AbstractKeycloakServerSupplier implements Supplier<Keycloa
 
         command = serverConfig.configure(command);
 
+        // Database startup and Keycloak connection setup
         if (requiresDatabase()) {
             instanceContext.getDependency(TestDatabase.class);
         }
 
+        // External Infinispan startup and Keycloak connection setup
+        if (command.isExternalInfinispanEnabled()) {
+            instanceContext.getDependency(InfinispanServer.class);
+        }
+
         ServerConfigInterceptorHelper interceptor = new ServerConfigInterceptorHelper(instanceContext.getRegistry());
         command = interceptor.intercept(command, instanceContext);
 
@@ -80,10 +86,6 @@ public abstract class AbstractKeycloakServerSupplier implements Supplier<Keycloa
 
     public abstract Logger getLogger();
 
-    protected String cache() {
-        return "local";
-    }
-
     @Override
     public int order() {
         return SupplierOrder.KEYCLOAK_SERVER;

test-framework/core/src/main/java/org/keycloak/testframework/server/DistributionKeycloakServer.java

@@ -4,7 +4,6 @@ import io.quarkus.maven.dependency.Dependency;
 import org.jboss.logging.Logger;
 import org.keycloak.it.utils.OutputConsumer;
 import org.keycloak.it.utils.RawKeycloakDistribution;
-import org.keycloak.testframework.config.Config;
 
 import java.nio.file.Path;
 import java.util.Collections;

test-framework/core/src/main/java/org/keycloak/testframework/server/KeycloakServerConfigBuilder.java

@@ -5,7 +5,7 @@ import io.quarkus.maven.dependency.DependencyBuilder;
 import io.smallrye.config.SmallRyeConfig;
 import org.eclipse.microprofile.config.spi.ConfigSource;
 import org.keycloak.common.Profile;
-import org.keycloak.common.Profile.Feature;
+import org.keycloak.testframework.infinispan.CacheType;
 
 import java.net.URISyntaxException;
 import java.nio.file.Path;
@@ -29,6 +29,8 @@ public class KeycloakServerConfigBuilder {
     private final LogBuilder log = new LogBuilder();
     private final Set<Dependency> dependencies = new HashSet<>();
     private final Set<Path> configFiles = new HashSet<>();
+    private CacheType cacheType = CacheType.LOCAL;
+    private boolean externalInfinispan = false;
 
     private KeycloakServerConfigBuilder(String command) {
         this.command = command;
@@ -48,8 +50,24 @@ public class KeycloakServerConfigBuilder {
                 .option("bootstrap-admin-password", password);
     }
 
-    public KeycloakServerConfigBuilder cache(String cache) {
-        return option("cache", cache);
+    public KeycloakServerConfigBuilder cache(CacheType cacheType) {
+        this.cacheType = cacheType;
+        return this;
+    }
+
+    public KeycloakServerConfigBuilder externalInfinispanEnabled(boolean enabled) {
+        if (enabled) {
+            this.externalInfinispan = true;
+            cache(CacheType.ISPN);
+        } else {
+            this.externalInfinispan = false;
+            cache(CacheType.LOCAL);
+        }
+        return this;
+    }
+
+    public boolean isExternalInfinispanEnabled() {
+        return this.externalInfinispan;
     }
 
     public LogBuilder log() {
@@ -185,12 +203,19 @@ public class KeycloakServerConfigBuilder {
     }
 
     List<String> toArgs() {
+        // Cache setup -> supported values: local or ispn
+        option("cache", cacheType.name().toLowerCase());
+
         log.build();
 
         List<String> args = new LinkedList<>();
         args.add(command);
         for (Map.Entry<String, String> e : options.entrySet()) {
-            args.add("--" + e.getKey() + "=" + e.getValue());
+            if (e.getKey().startsWith("-D")) {
+                args.add(e.getKey() + "=" + e.getValue());
+            } else {
+                args.add("--" + e.getKey() + "=" + e.getValue());
+            }
         }
         if (!features.isEmpty()) {
             args.add("--features=" + String.join(",", features));

test-framework/core/src/main/java/org/keycloak/testframework/util/ContainerImages.java

@@ -0,0 +1,21 @@
+package org.keycloak.testframework.util;
+
+import java.io.IOException;
+import java.util.Properties;
+
+public class ContainerImages {
+
+    public static String getContainerImageName(String containerName) {
+        return loadProperties().getProperty(containerName + ".container");
+    }
+
+    private static Properties loadProperties() {
+        Properties properties = new Properties();
+        try {
+            properties.load(ContainerImages.class.getResourceAsStream("containers.properties"));
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        return properties;
+    }
+}

test-framework/core/src/main/resources/org/keycloak/testframework/util/containers.properties

@@ -1,6 +1,10 @@
+## Database containers ##
 mysql.container=${mysql.container}
 postgres.container=${postgresql.container}
 mariadb.container=${mariadb.container}
 mssql.container=${mssql.container}
 oracle.container=${oracledb.container}
-tidb.container=${tidb.container}
+tidb.container=${tidb.container}
+
+## Infinispan container ##
+infinispan.container=${infinispan.container}

test-framework/db-mariadb/src/main/java/org/keycloak/testframework/database/MariaDBTestDatabase.java

@@ -1,6 +1,7 @@
 package org.keycloak.testframework.database;
 
 import org.jboss.logging.Logger;
+import org.keycloak.testframework.util.ContainerImages;
 import org.testcontainers.containers.JdbcDatabaseContainer;
 import org.testcontainers.containers.MariaDBContainer;
 import org.testcontainers.utility.DockerImageName;
@@ -13,7 +14,7 @@ class MariaDBTestDatabase extends AbstractContainerTestDatabase {
 
     @Override
     public JdbcDatabaseContainer<?> createContainer() {
-        return new MariaDBContainer<>(DockerImageName.parse(DatabaseProperties.getContainerImageName(NAME)).asCompatibleSubstituteFor(NAME)).withCommand("--character-set-server=utf8 --collation-server=utf8_unicode_ci");
+        return new MariaDBContainer<>(DockerImageName.parse(ContainerImages.getContainerImageName(NAME)).asCompatibleSubstituteFor(NAME)).withCommand("--character-set-server=utf8 --collation-server=utf8_unicode_ci");
     }
 
     @Override

test-framework/db-mssql/src/main/java/org/keycloak/testframework/database/MSSQLServerTestDatabase.java

@@ -1,8 +1,10 @@
 package org.keycloak.testframework.database;
 
 import org.jboss.logging.Logger;
+import org.keycloak.testframework.util.ContainerImages;
 import org.testcontainers.containers.JdbcDatabaseContainer;
 import org.testcontainers.containers.MSSQLServerContainer;
+import org.testcontainers.utility.DockerImageName;
 
 import java.util.List;
 
@@ -15,7 +17,7 @@ class MSSQLServerTestDatabase extends AbstractContainerTestDatabase {
     @SuppressWarnings("resource")
     @Override
     public JdbcDatabaseContainer<?> createContainer() {
-        return new MSSQLServerContainer<>(DatabaseProperties.getContainerImageName(NAME)).withPassword(getPassword()).withEnv("MSSQL_PID", "Express").acceptLicense();
+        return new MSSQLServerContainer<>(DockerImageName.parse(ContainerImages.getContainerImageName(NAME))).withPassword(getPassword()).withEnv("MSSQL_PID", "Express").acceptLicense();
     }
 
     @Override

test-framework/db-mysql/src/main/java/org/keycloak/testframework/database/MySQLTestDatabase.java

@@ -1,6 +1,7 @@
 package org.keycloak.testframework.database;
 
 import org.jboss.logging.Logger;
+import org.keycloak.testframework.util.ContainerImages;
 import org.testcontainers.containers.JdbcDatabaseContainer;
 import org.testcontainers.containers.MySQLContainer;
 import org.testcontainers.utility.DockerImageName;
@@ -13,7 +14,7 @@ class MySQLTestDatabase extends AbstractContainerTestDatabase {
 
     @Override
     public JdbcDatabaseContainer<?> createContainer() {
-        return new MySQLContainer<>(DockerImageName.parse(DatabaseProperties.getContainerImageName(NAME)).asCompatibleSubstituteFor(NAME));
+        return new MySQLContainer<>(DockerImageName.parse(ContainerImages.getContainerImageName(NAME)).asCompatibleSubstituteFor(NAME));
     }
 
     @Override

test-framework/db-oracle/src/main/java/org/keycloak/testframework/database/OracleTestDatabase.java

@@ -1,6 +1,7 @@
 package org.keycloak.testframework.database;
 
 import org.jboss.logging.Logger;
+import org.keycloak.testframework.util.ContainerImages;
 import org.testcontainers.containers.JdbcDatabaseContainer;
 import org.testcontainers.oracle.OracleContainer;
 import org.testcontainers.utility.DockerImageName;
@@ -13,7 +14,7 @@ class OracleTestDatabase extends AbstractContainerTestDatabase {
 
     @Override
     public JdbcDatabaseContainer<?> createContainer() {
-        return new OracleContainer(DockerImageName.parse(DatabaseProperties.getContainerImageName(NAME)).asCompatibleSubstituteFor("gvenzl/oracle-free"));
+        return new OracleContainer(DockerImageName.parse(ContainerImages.getContainerImageName(NAME)).asCompatibleSubstituteFor("gvenzl/oracle-free"));
     }
 
     @Override

test-framework/db-postgres/src/main/java/org/keycloak/testframework/database/PostgresTestDatabase.java

@@ -1,6 +1,7 @@
 package org.keycloak.testframework.database;
 
 import org.jboss.logging.Logger;
+import org.keycloak.testframework.util.ContainerImages;
 import org.testcontainers.containers.JdbcDatabaseContainer;
 import org.testcontainers.containers.PostgreSQLContainer;
 import org.testcontainers.utility.DockerImageName;
@@ -15,7 +16,7 @@ public class PostgresTestDatabase extends AbstractContainerTestDatabase {
 
     @Override
     public JdbcDatabaseContainer<?> createContainer() {
-        return new PostgreSQLContainer<>(DockerImageName.parse(DatabaseProperties.getContainerImageName(NAME)).asCompatibleSubstituteFor(NAME));
+        return new PostgreSQLContainer<>(DockerImageName.parse(ContainerImages.getContainerImageName(NAME)).asCompatibleSubstituteFor(NAME));
     }
 
     @Override

test-framework/db-tidb/src/main/java/org/keycloak/testframework/database/TiDBTestDatabase.java

@@ -2,6 +2,7 @@ package org.keycloak.testframework.database;
 
 import org.apache.commons.lang3.StringUtils;
 import org.jboss.logging.Logger;
+import org.keycloak.testframework.util.ContainerImages;
 import org.testcontainers.containers.JdbcDatabaseContainer;
 import org.testcontainers.tidb.TiDBContainer;
 import org.testcontainers.utility.DockerImageName;
@@ -14,7 +15,7 @@ class TiDBTestDatabase extends AbstractContainerTestDatabase {
 
     @Override
     public JdbcDatabaseContainer<?> createContainer() {
-        return new TiDBContainer(DockerImageName.parse(DatabaseProperties.getContainerImageName(NAME)).asCompatibleSubstituteFor("pingcap/tidb")){
+        return new TiDBContainer(DockerImageName.parse(ContainerImages.getContainerImageName(NAME)).asCompatibleSubstituteFor("pingcap/tidb")){
             @Override
             public TiDBContainer withDatabaseName(String databaseName) {
                 if(StringUtils.equals(this.getDatabaseName(), databaseName)) {