Add workflows and utils to review stability of testsuite (#40268)

Closes #40267

Signed-off-by: stianst <stianst@gmail.com>

.github/workflows/stability-base-reruns.yml

@@ -0,0 +1,59 @@
+name: Stability - Base Reruns
+
+on:
+  workflow_dispatch:
+    inputs:
+      tests:
+        type: string
+        description: Tests to run
+        required: true
+      count:
+        type: number
+        description: Number of re-runs
+        default: 50
+
+env:
+  MAVEN_ARGS: "-B -nsu -Daether.connector.http.connectionMaxTtl=25"
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Build Keycloak
+        uses: ./.github/actions/build-keycloak
+
+  base-integration-tests:
+    name: Base IT
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 360
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - id: integration-test-setup
+        name: Integration test setup
+        uses: ./.github/actions/integration-test-setup
+
+      - name: Run base tests
+        run: |
+          TESTS="${{ inputs.tests }}"
+          COUNT=${{ inputs.count }}
+          echo "Tests: $TESTS, count: $COUNT"
+          FAILURES=0
+          for i in $(seq 1 $COUNT); do
+            echo "========================================================================="
+            echo Run: $i
+            echo "========================================================================="
+            ./mvnw test ${{ env.SUREFIRE_RETRY }} -Pauth-server-quarkus -Dtest=$TESTS -pl testsuite/integration-arquillian/tests/base 2>&1 | misc/log/trimmer.sh || FAILURES=$(($FAILURES + 1)) 
+            FAILURES=$(($FAILURES + $?))
+          done
+          echo "Failures: $FAILURES"
+          exit $FAILURES

.github/workflows/stability-base.yml

@@ -0,0 +1,56 @@
+name: Stability - Base
+
+on:
+  workflow_dispatch:
+
+env:
+  MAVEN_ARGS: "-B -nsu -Daether.connector.http.connectionMaxTtl=25"
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Build Keycloak
+        uses: ./.github/actions/build-keycloak
+
+  base-integration-tests:
+    name: Base IT
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 100
+    strategy:
+      matrix:
+        group: [ 1, 2, 3, 4, 5, 6 ]
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - id: integration-test-setup
+        name: Integration test setup
+        uses: ./.github/actions/integration-test-setup
+
+      - name: Run base tests
+        run: |
+          TESTS=`testsuite/integration-arquillian/tests/base/testsuites/base-suite.sh ${{ matrix.group }}`
+          echo "Tests: $TESTS"
+          ./mvnw test ${{ env.SUREFIRE_RETRY }} -Pauth-server-quarkus -Dtest=$TESTS -pl testsuite/integration-arquillian/tests/base 2>&1 | misc/log/trimmer.sh
+
+  delete-artifacts:
+    name: Delete artifacts
+    needs: base-integration-tests
+    runs-on: ubuntu-latest
+    if: always()
+    env:
+      GH_TOKEN: ${{ github.token }}
+    steps:
+      - name: Delete artifacts
+        run:
+          gh api /repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts | jq .artifacts[].id | xargs -I {} gh api -X DELETE /repos/${{ github.repository }}/actions/artifacts/{}

.github/workflows/stability-clustering.yml

@@ -0,0 +1,52 @@
+name: Stability - Clustering
+
+on:
+  workflow_dispatch:
+
+env:
+  MAVEN_ARGS: "-B -nsu -Daether.connector.http.connectionMaxTtl=25"
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Build Keycloak
+        uses: ./.github/actions/build-keycloak
+
+  clustering-integration-tests:
+    name: Clustering IT
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 35
+    env:
+      MAVEN_OPTS: -Xmx1024m
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - id: integration-test-setup
+        name: Integration test setup
+        uses: ./.github/actions/integration-test-setup
+
+      - name: Run cluster tests
+        run: |
+          ./mvnw test ${{ env.SUREFIRE_RETRY }} -Pauth-server-cluster-quarkus,db-postgres "-Dwebdriver.chrome.driver=$CHROMEWEBDRIVER/chromedriver" -Dsession.cache.owners=2 -Dtest=**.cluster.** -pl testsuite/integration-arquillian/tests/base 2>&1 | misc/log/trimmer.sh
+
+  delete-artifacts:
+    name: Delete artifacts
+    needs: clustering-integration-tests
+    runs-on: ubuntu-latest
+    if: always()
+    env:
+      GH_TOKEN: ${{ github.token }}
+    steps:
+      - name: Delete artifacts
+        run:
+          gh api /repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts | jq .artifacts[].id | xargs -I {} gh api -X DELETE /repos/${{ github.repository }}/actions/artifacts/{}

.github/workflows/stability-js-ci.yml

@@ -0,0 +1,133 @@
+name: Stability - Keycloak JavaScript CI
+
+on:
+  workflow_dispatch:
+
+env:
+  MAVEN_ARGS: "-B -nsu -Daether.connector.http.connectionMaxTtl=25"
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  build-keycloak:
+    name: Build Keycloak
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Build Keycloak
+        uses: ./.github/actions/build-keycloak
+
+      - name: Prepare archive for upload
+        run: |
+          mv ./quarkus/dist/target/keycloak-999.0.0-SNAPSHOT.tar.gz ./keycloak-999.0.0-SNAPSHOT.tar.gz
+
+      - name: Upload Keycloak dist
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: keycloak
+          path: keycloak-999.0.0-SNAPSHOT.tar.gz
+
+  account-ui-e2e:
+    name: Account UI E2E
+    needs:
+      - build-keycloak
+    runs-on: ubuntu-latest
+    env:
+      WORKSPACE: "@keycloak/keycloak-account-ui"
+    strategy:
+      matrix:
+        browser: [chromium, firefox]
+        exclude:
+          # Only test with Firefox on scheduled runs
+          - browser: ${{ github.event_name != 'workflow_dispatch' && 'firefox' || '' }}
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - uses: ./.github/actions/pnpm-setup
+
+      - name: Download Keycloak server
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: keycloak
+
+      - name: Setup Java
+        uses: ./.github/actions/java-setup
+
+      - name: Start Keycloak server
+        run: |
+          tar xfvz keycloak-999.0.0-SNAPSHOT.tar.gz
+          keycloak-999.0.0-SNAPSHOT/bin/kc.sh start-dev --features=transient-users,oid4vc-vci &> ~/server.log &
+        env:
+          KC_BOOTSTRAP_ADMIN_USERNAME: admin
+          KC_BOOTSTRAP_ADMIN_PASSWORD: admin
+
+      - name: Install Playwright browsers
+        run: pnpm --fail-if-no-match --filter ${{ env.WORKSPACE }} exec playwright install --with-deps
+        working-directory: js
+
+      - name: Run Playwright tests
+        run: pnpm --fail-if-no-match --filter ${{ env.WORKSPACE }} test -- --project=${{ matrix.browser }}
+        working-directory: js
+
+  admin-ui-e2e:
+    name: Admin UI E2E
+    needs:
+      - build-keycloak
+    runs-on: ubuntu-latest
+    env:
+      WORKSPACE: "@keycloak/keycloak-admin-ui"
+    strategy:
+      matrix:
+        browser: [chromium, firefox]
+        exclude:
+          # Only test with Firefox on scheduled runs
+          - browser: ${{ github.event_name != 'workflow_dispatch' && 'firefox' || '' }}
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - uses: ./.github/actions/pnpm-setup
+
+      - name: Download Keycloak server
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: keycloak
+
+      - name: Setup Java
+        uses: ./.github/actions/java-setup
+
+      - name: Start Keycloak server
+        run: |
+          tar xfvz keycloak-999.0.0-SNAPSHOT.tar.gz
+          keycloak-999.0.0-SNAPSHOT/bin/kc.sh start-dev --features=admin-fine-grained-authz:v2,transient-users &> ~/server.log &
+        env:
+          KC_BOOTSTRAP_ADMIN_USERNAME: admin
+          KC_BOOTSTRAP_ADMIN_PASSWORD: admin
+          KC_BOOTSTRAP_ADMIN_CLIENT_ID: temporary-admin-service
+          KC_BOOTSTRAP_ADMIN_CLIENT_SECRET: temporary-admin-service
+
+      - name: Install Playwright browsers
+        run: pnpm --fail-if-no-match --filter ${{ env.WORKSPACE }} exec playwright install --with-deps
+        working-directory: js
+
+      - name: Run Playwright tests
+        run: pnpm --fail-if-no-match --filter ${{ env.WORKSPACE }} test:integration -- --project=${{ matrix.browser }}
+        working-directory: js
+
+  delete-artifacts:
+    name: Delete artifacts
+    needs:
+      - account-ui-e2e
+      - admin-ui-e2e
+    runs-on: ubuntu-latest
+    if: always()
+    env:
+      GH_TOKEN: ${{ github.token }}
+    steps:
+      - name: Delete artifacts
+        run:
+          gh api /repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts | jq .artifacts[].id | xargs -I {} gh api -X DELETE /repos/${{ github.repository }}/actions/artifacts/{}

misc/test-stability/.gitignore

@@ -0,0 +1 @@
+logs*

misc/test-stability/README.md

@@ -0,0 +1,43 @@
+# Test stability utils
+
+This directory contains a number of utils to schedule many runs on GitHub Actions and parsing the results. This is 
+useful to periodically do a sanity check of the test stability.
+
+Some dedicated GitHub Actions workflows are used for this purpose, as other workflows include test retries, and will
+also cancel multiple concurrent runs for the same workflow.
+
+Available workflows:
+
+* stability-base.yml
+* stability-clustering.yml
+* stability-js-ci.yml
+
+There is also another special workflow `stability-base-reruns.yml` this can be used to run a single test within the
+base testsuite many times, which can be useful when trying to fix instability in a specific test.
+
+The runs should not be scheduled in the main Keycloak organization, but rather in your own fork of Keycloak. This is to
+prevent impacting other developers and contributors.
+
+To schedule a run use `schedule-runs.sh`, for example:
+
+```
+./schedule-runs.sh -w stability-base.yml
+```
+
+Once scheduled you need to wait for all runs to complete, you can check the status with `status.sh`, for example:
+
+```
+./status.sh -w stability-base.yml
+```
+
+After all runs have completed you can download the logs for the failed runs using `download-logs.sh`, for example:
+
+```
+./download-logs.sh -w stability-base.yml
+```
+
+Final step is to parse the logs to get a report of failed tests using `parse-logs.sh`, for example:
+
+```
+./parse-logs.sh logs
+```

misc/test-stability/download-logs.sh

@@ -0,0 +1,61 @@
+#!/bin/bash -e
+
+function help() {
+     echo "Download logs for failed GitHub Actions runs"
+     echo
+     echo "options:"
+     echo "-b     Branch to use (defaults to main)"
+     echo "-w     Workflow name (required)"
+     echo "-d     Date range (defaults to today)"
+     echo "-l     Download directory (defaults to logs)"
+     echo
+}
+
+while getopts ":b:d:w:l:" option; do
+  case $option in
+    b)
+      BRANCH=$OPTARG;;
+    w)
+      WORKFLOW=$OPTARG;;
+    d)
+      DATE=$OPTARG;;
+    l)
+      LOGS=$OPTARG;;
+    *)
+      help
+      exit;;
+  esac
+done
+
+if [ "$DATE" == "" ]; then
+  DATE=$(date -Idate)
+fi
+
+USER=$(gh api user | jq -r '.login')
+
+if [ "$BRANCH" == "" ]; then
+  BRANCH="main"
+fi
+
+if [ "$WORKFLOW" == "" ]; then
+  echo -e "Error: Workflow not specified\n" && help && exit 1
+  exit 1
+fi
+
+if [ "$LOGS" == "" ]; then
+  LOGS="logs"
+fi
+
+if [ ! -d "$LOGS" ]; then
+    mkdir "$LOGS"
+fi
+
+for i in $(gh run list -L 100 -R "$USER/keycloak" -w "$WORKFLOW" -s failure --created "$DATE" --json databaseId | jq -r .[].databaseId); do
+    echo -n "($i) "
+    if [ ! -f "$LOGS/$i" ]; then
+        gh run -R "$USER/keycloak" view --log-failed "$i" > "$LOGS/$i"
+    fi
+    if [ ! -f "$LOGS/$i.json" ]; then
+        gh run -R "$USER/keycloak" view --json name,conclusion,databaseId,jobs "$i" > "$LOGS/$i.json"
+    fi
+done

misc/test-stability/parse-logs.sh

@@ -0,0 +1,10 @@
+#!/bin/bash -e
+
+LOG_DIR="$1"
+
+if [ "$LOG_DIR" == "" ]; then
+  echo "usage: parse-logs.sh <log directory>"
+  exit 1
+fi
+
+java -jar target/test-logs-parser.jar "$1" "$2"

misc/test-stability/pom.xml

@@ -0,0 +1,59 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.keycloak.test.stability</groupId>
+    <artifactId>test-logs-parser</artifactId>
+    <version>0.1-SNAPSHOT</version>
+
+    <name>Test Log Parser</name>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <maven.compiler.source>17</maven.compiler.source>
+        <maven.compiler.target>17</maven.compiler.target>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>2.19.0</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <finalName>${project.artifactId}</finalName>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <version>3.4.2</version>
+                <configuration>
+                    <archive>
+                        <manifest>
+                            <addClasspath>true</addClasspath>
+                            <mainClass>org.keycloak.test.logparser.TestReport</mainClass>
+                        </manifest>
+                    </archive>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.6.0</version>
+                <configuration>
+                    <dependencyReducedPomLocation>${build.directory}/dependency-reduced-pom.xml</dependencyReducedPomLocation>
+                </configuration>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>

misc/test-stability/schedule-runs.sh

@@ -0,0 +1,47 @@
+#!/bin/bash -e
+
+function help() {
+     echo "Schedule many runs for a GitHub Actions workflow"
+     echo
+     echo "options:"
+     echo "-b     Branch to use (defaults to main)"
+     echo "-w     Workflow name (required)"
+     echo "-n     Number of runs (defaults to 100)"
+     echo
+}
+
+while getopts ":b:n:w:" option; do
+  case $option in
+    b)
+      BRANCH=$OPTARG;;
+    w)
+      WORKFLOW=$OPTARG;;
+    n)
+      RUNS=$OPTARG;;
+    *)
+      help
+      exit;;
+  esac
+done
+
+if [ "$RUNS" == "" ]; then
+  RUNS=100
+fi
+
+if [ "$WORKFLOW" == "" ]; then
+  echo -e "Error: Workflow not specified\n" && help && exit 1
+fi
+
+if [ "$BRANCH" == "" ]; then
+  BRANCH="main"
+fi
+
+USER=$(gh api user | jq -r '.login')
+
+echo "Scheduling $RUNS run(s) in $USER/keycloak for workflow $WORKFLOW and branch $BRANCH"
+echo ""
+
+for i in $(seq 1 "$RUNS"); do
+    echo -n "($i) "
+    gh workflow run -R "$USER/keycloak" -r "$BRANCH" "$WORKFLOW"
+done

misc/test-stability/src/main/java/org/keycloak/test/logparser/GitHubRun.java

@@ -0,0 +1,16 @@
+package org.keycloak.test.logparser;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+import java.util.List;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public record GitHubRun(String name, String conclusion, String databaseId, List<GitHubRunJob> jobs) {
+
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    public record GitHubRunJob(String name, String conclusion, String databaseId, String url, List<GitHubRunJobStep> steps) {}
+
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    public record GitHubRunJobStep(String name, String conclusion, String databaseId) {}
+
+}

misc/test-stability/src/main/java/org/keycloak/test/logparser/LogFailure.java

@@ -0,0 +1,4 @@
+package org.keycloak.test.logparser;
+
+public record LogFailure(String job, String test) {
+}

misc/test-stability/src/main/java/org/keycloak/test/logparser/LogParser.java

@@ -0,0 +1,11 @@
+package org.keycloak.test.logparser;
+
+import java.util.List;
+
+public interface LogParser {
+
+    boolean supports(List<String> lines);
+
+    List<LogFailure> parseFailures(List<String> lines);
+
+}

misc/test-stability/src/main/java/org/keycloak/test/logparser/MarkdownReportGenerator.java

@@ -0,0 +1,40 @@
+package org.keycloak.test.logparser;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class MarkdownReportGenerator implements ReportGenerator {
+
+    public void printReport(List<GitHubRun> runs, List<TestFailure> failedTests) {
+        printRunSummary(runs);
+        printFailedTests(failedTests);
+    }
+
+    private void printRunSummary(List<GitHubRun> runs) {
+        List<RunFailure> failedRuns = Utils.toRunFailures(runs);
+
+        System.out.println("# Failed steps");
+        System.out.println("| Num | Step | Failures | ");
+        System.out.println("| --- | ---- | -------- | ");
+
+        for (RunFailure run : failedRuns) {
+            String failures = run.details().stream()
+                    .map(r -> "[" + r.runId() + "](" + r.url() + ")")
+                    .collect(Collectors.joining(" "));
+            System.out.println("| " + run.details().size() + " | " + run.run() + " | " + failures + " |");
+        }
+    }
+
+    public void printFailedTests(List<TestFailure> failedTests) {
+        System.out.println("# Failed tests");
+        System.out.println("| Num | Test | Failures | ");
+        System.out.println("| --- | ---- | -------- | ");
+        for (TestFailure failedTest : failedTests) {
+            String failures = failedTest.details().stream()
+                    .map(r -> "[" + r.runId() + "](" + r.url() + ")")
+                    .collect(Collectors.joining(" "));
+            System.out.println("| " + failedTest.details().size() + " | " + failedTest.test() + " | " + failures + " |");
+        }
+    }
+
+}

misc/test-stability/src/main/java/org/keycloak/test/logparser/PlaywrightLogParser.java

@@ -0,0 +1,34 @@
+package org.keycloak.test.logparser;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class PlaywrightLogParser implements LogParser {
+
+    private static final Pattern FAILURE = Pattern.compile("([^\\t]*)\\t.*##\\[error].*› (test/[^›]*).*");
+
+    @Override
+    public boolean supports(List<String> lines) {
+        return lines.stream().anyMatch(l -> l.contains("Playwright test"));
+    }
+
+    public List<LogFailure> parseFailures(List<String> lines) {
+        List<LogFailure> logFailures = new LinkedList<>();
+        for (String l : lines) {
+            Matcher m = FAILURE.matcher(l);
+
+            if (m.matches()) {
+                String job = m.group(1);
+                String test = m.group(2);
+
+                if (logFailures.stream().noneMatch(lf -> lf.job().equals(job) && lf.test().equals(test))) {
+                    logFailures.add(new LogFailure(job, test));
+                }
+            }
+        }
+        return logFailures;
+    }
+
+}

misc/test-stability/src/main/java/org/keycloak/test/logparser/ReportGenerator.java

@@ -0,0 +1,9 @@
+package org.keycloak.test.logparser;
+
+import java.util.List;
+
+public interface ReportGenerator {
+
+    void printReport(List<GitHubRun> runs, List<TestFailure> failedTests);
+
+}

misc/test-stability/src/main/java/org/keycloak/test/logparser/RunFailure.java

@@ -0,0 +1,9 @@
+package org.keycloak.test.logparser;
+
+import java.util.List;
+
+public record RunFailure(String run, List<FailedRunDetails> details) {
+
+    public record FailedRunDetails(String runName, String runId, String jobName, String jobId, String url) {}
+
+}

misc/test-stability/src/main/java/org/keycloak/test/logparser/SureFireLogParser.java

@@ -0,0 +1,28 @@
+package org.keycloak.test.logparser;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class SureFireLogParser implements LogParser {
+
+    private static final Pattern FAILURE = Pattern.compile("([^\\t]*)\\t.*\\[ERROR] {3}(.*Test[^ ]*).*");
+
+    @Override
+    public boolean supports(List<String> lines) {
+        return lines.stream().anyMatch(l -> l.contains("org.apache.maven.plugins:maven-surefire-plugin"));
+    }
+
+    public List<LogFailure> parseFailures(List<String> lines) {
+        List<LogFailure> logFailures = new LinkedList<>();
+        for (String l : lines) {
+            Matcher m = FAILURE.matcher(l);
+            if (m.matches()) {
+                logFailures.add(new LogFailure(m.group(1), m.group(2)));
+            }
+        }
+        return logFailures;
+    }
+
+}

misc/test-stability/src/main/java/org/keycloak/test/logparser/TestFailure.java

@@ -0,0 +1,9 @@
+package org.keycloak.test.logparser;
+
+import java.util.List;
+
+public record TestFailure(String test, List<FailedTestDetails> details) {
+
+    public record FailedTestDetails(String runName, String runId, String jobName, String jobId, String url) {}
+
+}

misc/test-stability/src/main/java/org/keycloak/test/logparser/TestReport.java

@@ -0,0 +1,67 @@
+package org.keycloak.test.logparser;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+public class TestReport {
+
+    private static final Map<String, ReportGenerator> REPORT_GENERATORS = Map.of(
+            "text", new TextReportGenerator(),
+            "md", new MarkdownReportGenerator()
+    );
+    private static final List<LogParser> LOG_PARSERS = List.of(new SureFireLogParser(), new PlaywrightLogParser());
+
+    public static void main(String[] args) throws IOException {
+        File logDirectory = new File(args[0]);
+        ReportGenerator reportGenerator = REPORT_GENERATORS.get(args.length == 2 && !args[1].isBlank() ? args[1] : "text");
+
+        List<GitHubRun> runs = loadRuns(logDirectory);
+        List<TestFailure> failedTests = loadFailedTests(logDirectory, runs);
+
+        reportGenerator.printReport(runs, failedTests);
+    }
+
+    private static List<GitHubRun> loadRuns(File logDirectory) throws IOException {
+        ObjectMapper objectMapper = new ObjectMapper();
+        List<GitHubRun> runs = new LinkedList<>();
+        for (File logFile : logDirectory.listFiles(f -> f.getName().endsWith(".json"))) {
+            runs.add(objectMapper.readValue(logFile, GitHubRun.class));
+        }
+        return runs;
+    }
+
+    private static List<TestFailure> loadFailedTests(File logDirectory, List<GitHubRun> runs) throws IOException {
+        List<TestFailure> failedTests = new LinkedList<>();
+
+        for (File logFile : logDirectory.listFiles(f -> !f.getName().endsWith(".json"))) {
+            List<String> lines = Utils.readLines(logFile);
+            Optional<LogParser> logParser = LOG_PARSERS.stream().filter(p -> p.supports(lines)).findFirst();
+
+            if (logParser.isPresent()) {
+                for (LogFailure logFailure : logParser.get().parseFailures(lines)) {
+                    TestFailure failedTest = failedTests.stream().filter(f -> f.test().equals(logFailure.test())).findFirst().or(() -> {
+                        TestFailure ft = new TestFailure(logFailure.test(), new LinkedList<>());
+                        failedTests.add(ft);
+                        return Optional.of(ft);
+                    }).get();
+
+                    String runId = logFile.getName();
+
+                    GitHubRun run = runs.stream().filter(r -> r.databaseId().equals(runId)).findFirst().get();
+                    GitHubRun.GitHubRunJob job = run.jobs().stream().filter(j -> j.name().equals(logFailure.job())).findFirst().get();
+
+                    failedTest.details().add(new TestFailure.FailedTestDetails(run.name(), run.databaseId(), job.name(), job.databaseId(), job.url()));
+                }
+            }
+        }
+
+        return failedTests;
+    }
+
+}

misc/test-stability/src/main/java/org/keycloak/test/logparser/TextReportGenerator.java

@@ -0,0 +1,43 @@
+package org.keycloak.test.logparser;
+
+import java.util.List;
+
+public class TextReportGenerator implements ReportGenerator {
+
+    public void printReport(List<GitHubRun> runs, List<TestFailure> failedTests) {
+        printDivider();
+        printRunSummary(runs);
+        printDivider();
+        printFailedTests(failedTests);
+        printDivider();
+    }
+
+    private void printRunSummary(List<GitHubRun> runs) {
+        List<RunFailure> failedRuns = Utils.toRunFailures(runs);
+
+        System.out.println("Failed steps:");
+        System.out.println();
+        for (RunFailure run : failedRuns) {
+            System.out.println(run.details().size() + "\t" + run.run());
+            for (RunFailure.FailedRunDetails details : run.details()) {
+                System.out.println("\t\t - " + details.url());
+            }
+        }
+    }
+
+    public void printFailedTests(List<TestFailure> failedTests) {
+        System.out.println("Failed tests:");
+        System.out.println();
+        for (TestFailure failedTest : failedTests) {
+            System.out.println(failedTest.details().size() + "\t" + failedTest.test());
+            for (TestFailure.FailedTestDetails details : failedTest.details()) {
+                System.out.println("\t\t - " + details.url());
+            }
+        }
+    }
+
+    public void printDivider() {
+        System.out.println("--------------------------------------------------------------------------------");
+    }
+
+}

misc/test-stability/src/main/java/org/keycloak/test/logparser/Utils.java

@@ -0,0 +1,39 @@
+package org.keycloak.test.logparser;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Optional;
+
+public class Utils {
+
+    public static List<String> readLines(File file) throws IOException {
+        return Files.lines(file.toPath()).toList();
+    }
+
+    public static List<RunFailure> toRunFailures(List<GitHubRun> runs) {
+        List<RunFailure> failedRuns = new LinkedList<>();
+        for (GitHubRun run : runs) {
+            for (GitHubRun.GitHubRunJob job : run.jobs()) {
+                if (job.conclusion().equals("failure")) {
+                    for (GitHubRun.GitHubRunJobStep step : job.steps()) {
+                        if (step.conclusion().equals("failure")) {
+                            String fullName = run.name() + " / " + job.name() + " / " + step.name();
+                            RunFailure failedRun = failedRuns.stream().filter(f -> f.run().equals(fullName)).findFirst().or(() -> {
+                                RunFailure fr = new RunFailure(fullName, new LinkedList<>());
+                                failedRuns.add(fr);
+                                return Optional.of(fr);
+                            }).get();
+
+                            failedRun.details().add(new RunFailure.FailedRunDetails(run.name(), run.databaseId(), job.name(), job.databaseId(), job.url()));
+                        }
+                    }
+                }
+            }
+        }
+        return failedRuns;
+    }
+
+}

misc/test-stability/status.sh

@@ -0,0 +1,44 @@
+#!/bin/bash -e
+
+function help() {
+     echo "View status of GitHub Actions runs"
+     echo
+     echo "options:"
+     echo "-b     Branch to use (defaults to main)"
+     echo "-w     Workflow name (required)"
+     echo "-d     Date range (defaults to today)"
+     echo
+}
+
+while getopts ":b:d:w:" option; do
+  case $option in
+    b)
+      BRANCH=$OPTARG;;
+    w)
+      WORKFLOW=$OPTARG;;
+    d)
+      DATE=$OPTARG;;
+    *)
+      help
+      exit;;
+  esac
+done
+
+if [ "$DATE" == "" ]; then
+  DATE=$(date -Idate)
+fi
+
+if [ "$BRANCH" == "" ]; then
+  BRANCH="main"
+fi
+
+if [ "$WORKFLOW" == "" ]; then
+  echo -e "Error: Workflow not specified\n" && help && exit 1
+fi
+
+USER=$(gh api user | jq -r '.login')
+
+echo "Status of $WORKFLOW in $USER/keycloak for branch $BRANCH"
+echo ""
+
+gh api -X GET "/repos/$USER/keycloak/actions/workflows/$WORKFLOW/runs" -F branch="$BRANCH" -F per_page=100 --paginate -F created="$DATE" | jq -r '.workflow_runs[] | [.status, .conclusion] | @csv' | sort | uniq -c