KEYCLOAK-5691 Galera cluster, full testsuite

testsuite/integration-arquillian/servers/auth-server/jboss/pom.xml

@@ -855,5 +855,23 @@
             </modules>
         </profile>
 
+
+        <profile>
+            <id>db-failover-mariadb</id>
+            <properties>
+                <jdbc.mvn.groupId>org.mariadb.jdbc</jdbc.mvn.groupId>
+                <jdbc.mvn.artifactId>mariadb-java-client</jdbc.mvn.artifactId>
+                <jdbc.mvn.version>2.0.3</jdbc.mvn.version>
+                <keycloak.connectionsJpa.user>keycloak</keycloak.connectionsJpa.user>
+                <keycloak.connectionsJpa.password>keycloak</keycloak.connectionsJpa.password>
+                
+                <mariadb.ha.mode>replication</mariadb.ha.mode>
+                <mariadb.hosts>localhost:3316,localhost:3326</mariadb.hosts>
+                <mariadb.database>keycloak</mariadb.database>
+                <mariadb.options></mariadb.options>
+                <keycloak.connectionsJpa.url>jdbc:mariadb:${mariadb.ha.mode}://${mariadb.hosts}/${mariadb.database}${mariadb.options}</keycloak.connectionsJpa.url>
+            </properties>
+        </profile>
+
     </profiles>
 </project>

testsuite/performance/db-failover/README.md

@@ -0,0 +1,67 @@
+# DB Failover Testing Utilities
+
+A set of scripts for testing DB failover scenarios.
+
+The scripts expect to be run with relative execution prefix `./` from within the `db-failover` directory.
+
+Provisioned services are defined in `../docker-compose-db-failover.yml` template.
+
+
+## Set the size of DB cluster
+
+Default size is 2 nodes. For a 3-node cluster run: `export NODES=3` before executing any of the scripts.
+
+For more than 3 nodes more service definitions need to be added to the docker-compose template.
+
+## Set up the environment
+
+Run `./setup.sh`
+
+This script will:
+1. Start a bootstrap DB instance of MariaDB cluster
+2. Start additional DB instances connected to the bootstrapped cluster
+3. Stop the bootstrap DB instance
+4. Optionally start Keycloak server
+
+Parameterized by environment variables:
+- `MARIADB_HA_MODE` See: [MariaDB HA parameters](https://mariadb.com/kb/en/library/failover-and-high-availability-with-mariadb-connector-j/#failover-high-availability-parameters)
+   Defaults to `replication`.
+- `MARIADB_OPTIONS` See: [MariaDB HA options](https://mariadb.com/kb/en/library/failover-and-high-availability-with-mariadb-connector-j/#failover-high-availability-options).
+   Use format: `?option1=value1[&option2=value2]...`. Default is an empty string.
+- `START_KEYCLOAK` Default is `false`. Use `export START_KEYCLOAK=true` to enable.
+
+More options relevant to MariaDB clustering can be found in `../db/mariadb/wsrep.cnf`.
+
+
+## Test the failover
+
+### Manual failover
+
+To induce a failure of specific DB node run: `./kill-node.sh X` where `X ∈ {1..3}`
+
+To reconnect the node back run: `./reconnect-node.sh X`
+
+
+### Automated failover loop
+
+Run `./loop.sh`
+
+This script will run an infinite loop of failover/failback of DB nodes, switching to the next node in each loop.
+
+Parameterized by environment variables:
+- `TIME_BETWEEN_FAILURES` Default is `60` (seconds).
+- `FAILURE_DURATION` Default is `60` (seconds).
+
+To exit the script press `Ctrl+C`.
+
+
+### Check number of table rows across the cluster
+
+Run: `./check-rows.sh`
+
+
+## Tear down the environment
+
+Run `./teardown.sh`
+
+This will stop all services and delete the database.

testsuite/performance/db-failover/check-rows.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+. ./common.sh
+
+CONCAT_SQL=$(< db-failover/concat.sql)
+CONCAT_SQL_COMMAND='mysql -N -B -u keycloak --password=keycloak -e "$CONCAT_SQL" keycloak'
+ROWS_SQL=$(eval docker-compose -f docker-compose-db-failover.yml exec mariadb_1 $CONCAT_SQL_COMMAND | tr -dc '[:print:]')
+ROWS_SQL=${ROWS_SQL%UNION }
+ROWS_SQL_COMMAND='mysql -u keycloak --password=keycloak -e "$ROWS_SQL" keycloak'
+
+for (( i=1; i <= $NODES; i++)); do
+    ROWS[i]=$(eval docker-compose -f docker-compose-db-failover.yml exec mariadb_$i $ROWS_SQL_COMMAND)
+done
+
+DIFF=0
+for (( i=2; i <= $NODES; i++)); do
+    echo Node 1 vs Node $(( i )):
+    diff -y --suppress-common-lines <(echo "${ROWS[1]}") <(echo "${ROWS[i]}")
+    if [ $? -eq 0 ]; then echo No difference.; else DIFF=1; fi
+done
+
+exit $DIFF

testsuite/performance/db-failover/common.sh

@@ -0,0 +1,18 @@
+function killNode {
+    echo Killing mariadb_${1}
+    docker-compose -f docker-compose-db-failover.yml kill mariadb_${1}
+}
+
+function reconnectNode {
+    N=$1
+    NR=$(( N + 1 )); if [ "$NR" -gt "$NODES" ]; then NR=1; fi
+    export MARIADB_RUNNING_HOST=mariadb_${NR}
+    echo Attempting failback of mariadb_${N}, connecting to running cluster member mariadb_${NR}
+    docker-compose -f docker-compose-db-failover.yml up -d mariadb_${N} 
+}
+
+if [ -z $NODES ]; then export NODES=2; fi
+if [ -z $MARIADB_OPTIONS ]; then export MARIADB_OPTIONS=""; fi
+if [ -z $START_KEYCLOAK ]; then export START_KEYCLOAK=false; fi
+
+cd ..

testsuite/performance/db-failover/concat.sql

@@ -0,0 +1,3 @@
+SELECT CONCAT(
+    'SELECT "', table_name, '" AS table_name, COUNT(*) AS exact_row_count FROM `', table_schema, '`.`', table_name, '` UNION '
+) FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = 'keycloak';

testsuite/performance/db-failover/kill-node.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+. ./common.sh
+
+if [ -z $1 ]; then echo "Specify DB node to kill."; exit 1; fi
+
+killNode $1

testsuite/performance/db-failover/loop.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+. ./common.sh
+
+if [ -z "$TIME_BETWEEN_FAILURES" ]; then export TIME_BETWEEN_FAILURES=60; fi
+if [ -z "$FAILURE_DURATION" ]; then export FAILURE_DURATION=60; fi
+
+echo Running DB failover loop with the following parameters:
+echo NODES=$NODES
+echo TIME_BETWEEN_FAILURES=$TIME_BETWEEN_FAILURES
+echo FAILURE_DURATION=$FAILURE_DURATION
+echo
+echo Press Ctrl+C to interrupt.
+echo
+
+N=1
+
+while :
+do
+    
+    killNode $N
+
+    echo Waiting $FAILURE_DURATION s before attempting to reconnect mariadb_${N}
+    sleep $FAILURE_DURATION
+
+    reconnectNode $N
+
+    echo Waiting $TIME_BETWEEN_FAILURES s before inducing another failure.
+    echo
+    sleep $TIME_BETWEEN_FAILURES
+
+    N=$((N+1))
+    if [ "$N" -gt "$NODES" ]; then N=1; fi
+
+done

testsuite/performance/db-failover/reconnect-node.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+. ./common.sh
+
+if [ -z $1 ]; then echo "Specify DB node to reconnect to cluster."; exit 1; fi
+
+reconnectNode $1

testsuite/performance/db-failover/setup.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+. ./common.sh
+
+if [ -z "$DB_BOOTSTRAP_TIMEOUT" ]; then DB_BOOTSTRAP_TIMEOUT=10; fi
+if [ -z "$DB_JOIN_TIMEOUT" ]; then DB_JOIN_TIMEOUT=5; fi
+
+echo Setting up Keycloak DB failover environment:
+
+echo Starting DB bootstrap instance.
+docker-compose -f docker-compose-db-failover.yml up -d --build mariadb_bootstrap
+echo Waiting $DB_BOOTSTRAP_TIMEOUT s for the DB to initialize.
+sleep $DB_BOOTSTRAP_TIMEOUT
+
+MARIADB_HOSTS=""
+for (( i=1; i<=$NODES; i++ )); do
+
+    MARIADB_HOSTS=$MARIADB_HOSTS,mariadb_$i:3306
+
+    echo Starting DB node $i.
+    docker-compose -f docker-compose-db-failover.yml up -d mariadb_$i
+    echo Waiting $DB_JOIN_TIMEOUT s for the DB node to join
+    echo
+    sleep $DB_JOIN_TIMEOUT
+
+done
+
+echo Turning off the DB bootstrap instance.
+docker-compose -f docker-compose-db-failover.yml stop mariadb_bootstrap
+
+export MARIADB_HOSTS=${MARIADB_HOSTS/,/}
+echo MARIADB_HOSTS=$MARIADB_HOSTS
+
+if $START_KEYCLOAK; then
+    echo Starting Keycloak server.
+    docker-compose -f docker-compose-db-failover.yml up -d --build keycloak
+    ./healthcheck.sh
+fi

testsuite/performance/db-failover/teardown.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+. ./common.sh
+
+echo Stopping Keycloak DB failover environment.
+
+docker-compose -f docker-compose-db-failover.yml down -v
+

testsuite/performance/db/mariadb/wsrep.cnf

@@ -15,7 +15,12 @@ innodb_buffer_pool_size=122M
 
 wsrep_on=ON
 wsrep_provider=/usr/lib/galera/libgalera_smm.so
+
 wsrep_provider_options="gcache.size=300M; gcache.page_size=300M"
+#wsrep_provider_options="gcache.size=300M; gcache.page_size=300M; pc.bootstrap=YES"
+#wsrep_provider_options="gcache.size=300M; gcache.page_size=300M; pc.bootstrap=YES; pc.ignore_sb=TRUE"
+# See: http://galeracluster.com/documentation-webpages/twonode.html
+
 wsrep_cluster_address="gcomm://"
 wsrep_cluster_name="galera_cluster_keycloak"
 wsrep_sst_method=rsync

testsuite/performance/docker-compose-cluster.yml

@@ -41,7 +41,7 @@ services:
             CONFIGURATION: standalone-ha.xml
             PUBLIC_SUBNET: 10.0.1.0/24
             PRIVATE_SUBNET: 10.0.1.0/24
-            MARIADB_HOST: mariadb
+            MARIADB_HOSTS: mariadb:3306
             MARIADB_DATABASE: keycloak
             MARIADB_USER: keycloak
             MARIADB_PASSWORD: keycloak

testsuite/performance/docker-compose-crossdc.yml

@@ -131,7 +131,7 @@ services:
             CONFIGURATION: standalone-ha.xml
             PUBLIC_SUBNET: 10.1.1.0/24
             PRIVATE_SUBNET: 10.1.1.0/24
-            MARIADB_HOST: mariadb_dc1
+            MARIADB_HOSTS: mariadb_dc1:3306
             MARIADB_DATABASE: keycloak
             MARIADB_USER: keycloak
             MARIADB_PASSWORD: keycloak
@@ -172,7 +172,7 @@ services:
             CONFIGURATION: standalone-ha.xml
             PUBLIC_SUBNET: 10.2.1.0/24
             PRIVATE_SUBNET: 10.2.1.0/24
-            MARIADB_HOST: mariadb_dc2
+            MARIADB_HOSTS: mariadb_dc2:3306
             MARIADB_DATABASE: keycloak
             MARIADB_USER: keycloak
             MARIADB_PASSWORD: keycloak

testsuite/performance/docker-compose-db-failover.yml

@@ -0,0 +1,101 @@
+version: "2.2"
+
+networks:
+    keycloak:
+        ipam:
+            config:
+            - subnet: 10.0.1.0/24
+
+    db_replication:
+        ipam:
+            config:
+            - subnet: 10.0.3.0/24
+        
+services:
+
+    mariadb_bootstrap:
+        build: db/mariadb
+        image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest}
+        networks:
+            - db_replication
+        environment:
+            MYSQL_ROOT_PASSWORD: root
+            MYSQL_INITDB_SKIP_TZINFO: foo
+            MYSQL_DATABASE: keycloak
+            MYSQL_USER: keycloak
+            MYSQL_PASSWORD: keycloak
+        entrypoint: docker-entrypoint-wsrep.sh
+        command: --wsrep-new-cluster
+            
+    mariadb_1:
+        build: db/mariadb
+        image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest}
+        networks:
+            - db_replication
+            - keycloak
+        environment:
+            MYSQL_ROOT_PASSWORD: root
+            MYSQL_INITDB_SKIP_TZINFO: foo
+        entrypoint: docker-entrypoint-wsrep.sh
+        command: --wsrep_cluster_address=gcomm://${MARIADB_RUNNING_HOST:-mariadb_bootstrap}
+        ports:
+            - "3316:3306"
+
+    mariadb_2:
+        build: db/mariadb
+        image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest}
+        networks:
+            - db_replication
+            - keycloak
+        environment:
+            MYSQL_ROOT_PASSWORD: root
+            MYSQL_INITDB_SKIP_TZINFO: foo
+        entrypoint: docker-entrypoint-wsrep.sh
+        command: --wsrep_cluster_address=gcomm://${MARIADB_RUNNING_HOST:-mariadb_bootstrap}
+        ports:
+            - "3326:3306"
+
+    mariadb_3:
+        build: db/mariadb
+        image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest}
+        networks:
+            - db_replication
+            - keycloak
+        environment:
+            MYSQL_ROOT_PASSWORD: root
+            MYSQL_INITDB_SKIP_TZINFO: foo
+        entrypoint: docker-entrypoint-wsrep.sh
+        command: --wsrep_cluster_address=gcomm://${MARIADB_RUNNING_HOST:-mariadb_bootstrap}
+        ports:
+            - "3336:3306"
+
+
+    keycloak:
+        build: keycloak
+        image: keycloak_test_keycloak:${KEYCLOAK_VERSION:-latest}
+        networks:
+            - keycloak
+        environment:
+            MARIADB_HA_MODE: ${MARIADB_HA_MODE:-replication}
+            MARIADB_HOSTS: ${MARIADB_HOSTS:-mariadb_1:3306,mariadb_2:3306}
+            MARIADB_OPTIONS: ${MARIADB_OPTIONS}
+            MARIADB_DATABASE: keycloak
+            MARIADB_USER: keycloak
+            MARIADB_PASSWORD: keycloak
+            KEYCLOAK_USER: admin
+            KEYCLOAK_PASSWORD: admin
+            # docker-compose syntax note: ${ENV_VAR:-<DEFAULT_VALUE>}
+            JAVA_OPTS: ${KEYCLOAK_JVM_MEMORY:--Xms64m -Xmx2g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true
+            HTTP_MAX_CONNECTIONS: ${KEYCLOAK_HTTP_MAX_CONNECTIONS:-500}
+            WORKER_IO_THREADS: ${KEYCLOAK_WORKER_IO_THREADS:-2}
+            WORKER_TASK_MAX_THREADS: ${KEYCLOAK_WORKER_TASK_MAX_THREADS:-16}
+            DS_MIN_POOL_SIZE: ${KEYCLOAK_DS_MIN_POOL_SIZE:-10}
+            DS_MAX_POOL_SIZE: ${KEYCLOAK_DS_MAX_POOL_SIZE:-100}
+            DS_POOL_PREFILL: "${KEYCLOAK_DS_POOL_PREFILL:-true}"
+            DS_PS_CACHE_SIZE: ${KEYCLOAK_DS_PS_CACHE_SIZE:-100}
+        ports:
+            - "8080:8080"
+            - "8443:8443"
+            - "7989:7989"
+            - "9990:9990"
+            - "9999:9999"

testsuite/performance/docker-compose.yml

@@ -35,7 +35,7 @@ services:
         networks:
             - keycloak
         environment:
-            MARIADB_HOST: mariadb
+            MARIADB_HOSTS: mariadb:3306
             MARIADB_DATABASE: keycloak
             MARIADB_USER: keycloak
             MARIADB_PASSWORD: keycloak

testsuite/performance/keycloak/configs/set-keycloak-ds-batch.cli

@@ -2,7 +2,7 @@
 
 cd /subsystem=datasources/data-source=KeycloakDS
 
-:write-attribute(name=connection-url, value=jdbc:mariadb://${env.MARIADB_HOST:mariadb}:${env.MARIADB_PORT:3306}/${env.MARIADB_DATABASE:keycloak})
+:write-attribute(name=connection-url, value=jdbc:mariadb:${env.MARIADB_HA_MODE:}://${env.MARIADB_HOSTS:mariadb:3306}/${env.MARIADB_DATABASE:keycloak}${env.MARIADB_OPTIONS:})
 :write-attribute(name=driver-name, value=mariadb)
 :write-attribute(name=user-name, value=${env.MARIADB_USER:keycloak})
 :write-attribute(name=password, value=${env.MARIADB_PASSWORD:keycloak})

testsuite/performance/keycloak/pom.xml

@@ -34,6 +34,12 @@
         Uses maven-dependency-plugin to unpack keycloak-server-dist artifact into `target/keycloak` which is then added to Docker image.
     </description>
     
+    <properties>
+        <server.groupId>org.keycloak</server.groupId>
+        <server.artifactId>keycloak-server-dist</server.artifactId>
+        <server.unpacked.folder>keycloak-${product.version}</server.unpacked.folder>
+    </properties>
+
     <build>
 
         <plugins>
@@ -51,8 +57,8 @@
                         <configuration>
                             <artifactItems>
                                 <artifactItem>
-                                    <groupId>org.keycloak</groupId>
+                                    <groupId>${server.groupId}</groupId>
-                                    <artifactId>keycloak-server-dist</artifactId>
+                                    <artifactId>${server.artifactId}</artifactId>
                                     <version>${project.version}</version>
                                     <type>zip</type>
                                     <outputDirectory>${project.build.directory}</outputDirectory>
@@ -77,7 +83,7 @@
                         <configuration>
                             <target>
                                 <move todir="${project.build.directory}/keycloak" failonerror="false">
-                                    <fileset dir="${project.build.directory}/keycloak-${project.version}"/>
+                                    <fileset dir="${project.build.directory}/${server.unpacked.folder}"/>
                                 </move>
                             </target>
                         </configuration>
@@ -87,4 +93,16 @@
         </plugins>
     </build>
     
+    <profiles>
+        
+        <profile>
+            <id>integration-testsuite-server</id>
+            <properties>
+                <server.groupId>org.keycloak.testsuite</server.groupId>
+                <server.artifactId>integration-arquillian-servers-auth-server-wildfly</server.artifactId>
+                <server.unpacked.folder>auth-server-wildfly</server.unpacked.folder>
+            </properties>
+        </profile>
+    </profiles>
+    
 </project>