KEYCLOAK-5691 Galera cluster, full testsuite
This commit is contained in:
parent
9e2ab2750a
commit
a45a2acc4c
17 changed files with 356 additions and 9 deletions
|
@ -855,5 +855,23 @@
|
|||
</modules>
|
||||
</profile>
|
||||
|
||||
|
||||
<profile>
|
||||
<id>db-failover-mariadb</id>
|
||||
<properties>
|
||||
<jdbc.mvn.groupId>org.mariadb.jdbc</jdbc.mvn.groupId>
|
||||
<jdbc.mvn.artifactId>mariadb-java-client</jdbc.mvn.artifactId>
|
||||
<jdbc.mvn.version>2.0.3</jdbc.mvn.version>
|
||||
<keycloak.connectionsJpa.user>keycloak</keycloak.connectionsJpa.user>
|
||||
<keycloak.connectionsJpa.password>keycloak</keycloak.connectionsJpa.password>
|
||||
|
||||
<mariadb.ha.mode>replication</mariadb.ha.mode>
|
||||
<mariadb.hosts>localhost:3316,localhost:3326</mariadb.hosts>
|
||||
<mariadb.database>keycloak</mariadb.database>
|
||||
<mariadb.options></mariadb.options>
|
||||
<keycloak.connectionsJpa.url>jdbc:mariadb:${mariadb.ha.mode}://${mariadb.hosts}/${mariadb.database}${mariadb.options}</keycloak.connectionsJpa.url>
|
||||
</properties>
|
||||
</profile>
|
||||
|
||||
</profiles>
|
||||
</project>
|
||||
|
|
67
testsuite/performance/db-failover/README.md
Normal file
67
testsuite/performance/db-failover/README.md
Normal file
|
@ -0,0 +1,67 @@
|
|||
# DB Failover Testing Utilities
|
||||
|
||||
A set of scripts for testing DB failover scenarios.
|
||||
|
||||
The scripts expect to be run with relative execution prefix `./` from within the `db-failover` directory.
|
||||
|
||||
Provisioned services are defined in `../docker-compose-db-failover.yml` template.
|
||||
|
||||
|
||||
## Set the size of DB cluster
|
||||
|
||||
Default size is 2 nodes. For a 3-node cluster run: `export NODES=3` before executing any of the scripts.
|
||||
|
||||
For more than 3 nodes more service definitions need to be added to the docker-compose template.
|
||||
|
||||
## Set up the environment
|
||||
|
||||
Run `./setup.sh`
|
||||
|
||||
This script will:
|
||||
1. Start a bootstrap DB instance of MariaDB cluster
|
||||
2. Start additional DB instances connected to the bootstrapped cluster
|
||||
3. Stop the bootstrap DB instance
|
||||
4. Optionally start Keycloak server
|
||||
|
||||
Parameterized by environment variables:
|
||||
- `MARIADB_HA_MODE` See: [MariaDB HA parameters](https://mariadb.com/kb/en/library/failover-and-high-availability-with-mariadb-connector-j/#failover-high-availability-parameters)
|
||||
Defaults to `replication`.
|
||||
- `MARIADB_OPTIONS` See: [MariaDB HA options](https://mariadb.com/kb/en/library/failover-and-high-availability-with-mariadb-connector-j/#failover-high-availability-options).
|
||||
Use format: `?option1=value1[&option2=value2]...`. Default is an empty string.
|
||||
- `START_KEYCLOAK` Default is `false`. Use `export START_KEYCLOAK=true` to enable.
|
||||
|
||||
More options relevant to MariaDB clustering can be found in `../db/mariadb/wsrep.cnf`.
|
||||
|
||||
|
||||
## Test the failover
|
||||
|
||||
### Manual failover
|
||||
|
||||
To induce a failure of specific DB node run: `./kill-node.sh X` where `X ∈ {1..3}`
|
||||
|
||||
To reconnect the node back run: `./reconnect-node.sh X`
|
||||
|
||||
|
||||
### Automated failover loop
|
||||
|
||||
Run `./loop.sh`
|
||||
|
||||
This script will run an infinite loop of failover/failback of DB nodes, switching to the next node in each loop.
|
||||
|
||||
Parameterized by environment variables:
|
||||
- `TIME_BETWEEN_FAILURES` Default is `60` (seconds).
|
||||
- `FAILURE_DURATION` Default is `60` (seconds).
|
||||
|
||||
To exit the script press `Ctrl+C`.
|
||||
|
||||
|
||||
### Check number of table rows across the cluster
|
||||
|
||||
Run: `./check-rows.sh`
|
||||
|
||||
|
||||
## Tear down the environment
|
||||
|
||||
Run `./teardown.sh`
|
||||
|
||||
This will stop all services and delete the database.
|
22
testsuite/performance/db-failover/check-rows.sh
Executable file
22
testsuite/performance/db-failover/check-rows.sh
Executable file
|
@ -0,0 +1,22 @@
|
|||
#!/bin/bash
|
||||
|
||||
. ./common.sh
|
||||
|
||||
CONCAT_SQL=$(< db-failover/concat.sql)
|
||||
CONCAT_SQL_COMMAND='mysql -N -B -u keycloak --password=keycloak -e "$CONCAT_SQL" keycloak'
|
||||
ROWS_SQL=$(eval docker-compose -f docker-compose-db-failover.yml exec mariadb_1 $CONCAT_SQL_COMMAND | tr -dc '[:print:]')
|
||||
ROWS_SQL=${ROWS_SQL%UNION }
|
||||
ROWS_SQL_COMMAND='mysql -u keycloak --password=keycloak -e "$ROWS_SQL" keycloak'
|
||||
|
||||
for (( i=1; i <= $NODES; i++)); do
|
||||
ROWS[i]=$(eval docker-compose -f docker-compose-db-failover.yml exec mariadb_$i $ROWS_SQL_COMMAND)
|
||||
done
|
||||
|
||||
DIFF=0
|
||||
for (( i=2; i <= $NODES; i++)); do
|
||||
echo Node 1 vs Node $(( i )):
|
||||
diff -y --suppress-common-lines <(echo "${ROWS[1]}") <(echo "${ROWS[i]}")
|
||||
if [ $? -eq 0 ]; then echo No difference.; else DIFF=1; fi
|
||||
done
|
||||
|
||||
exit $DIFF
|
18
testsuite/performance/db-failover/common.sh
Executable file
18
testsuite/performance/db-failover/common.sh
Executable file
|
@ -0,0 +1,18 @@
|
|||
function killNode {
|
||||
echo Killing mariadb_${1}
|
||||
docker-compose -f docker-compose-db-failover.yml kill mariadb_${1}
|
||||
}
|
||||
|
||||
function reconnectNode {
|
||||
N=$1
|
||||
NR=$(( N + 1 )); if [ "$NR" -gt "$NODES" ]; then NR=1; fi
|
||||
export MARIADB_RUNNING_HOST=mariadb_${NR}
|
||||
echo Attempting failback of mariadb_${N}, connecting to running cluster member mariadb_${NR}
|
||||
docker-compose -f docker-compose-db-failover.yml up -d mariadb_${N}
|
||||
}
|
||||
|
||||
if [ -z $NODES ]; then export NODES=2; fi
|
||||
if [ -z $MARIADB_OPTIONS ]; then export MARIADB_OPTIONS=""; fi
|
||||
if [ -z $START_KEYCLOAK ]; then export START_KEYCLOAK=false; fi
|
||||
|
||||
cd ..
|
3
testsuite/performance/db-failover/concat.sql
Normal file
3
testsuite/performance/db-failover/concat.sql
Normal file
|
@ -0,0 +1,3 @@
|
|||
SELECT CONCAT(
|
||||
'SELECT "', table_name, '" AS table_name, COUNT(*) AS exact_row_count FROM `', table_schema, '`.`', table_name, '` UNION '
|
||||
) FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = 'keycloak';
|
7
testsuite/performance/db-failover/kill-node.sh
Executable file
7
testsuite/performance/db-failover/kill-node.sh
Executable file
|
@ -0,0 +1,7 @@
|
|||
#!/bin/bash
|
||||
|
||||
. ./common.sh
|
||||
|
||||
if [ -z $1 ]; then echo "Specify DB node to kill."; exit 1; fi
|
||||
|
||||
killNode $1
|
35
testsuite/performance/db-failover/loop.sh
Executable file
35
testsuite/performance/db-failover/loop.sh
Executable file
|
@ -0,0 +1,35 @@
|
|||
#!/bin/bash
|
||||
|
||||
. ./common.sh
|
||||
|
||||
if [ -z "$TIME_BETWEEN_FAILURES" ]; then export TIME_BETWEEN_FAILURES=60; fi
|
||||
if [ -z "$FAILURE_DURATION" ]; then export FAILURE_DURATION=60; fi
|
||||
|
||||
echo Running DB failover loop with the following parameters:
|
||||
echo NODES=$NODES
|
||||
echo TIME_BETWEEN_FAILURES=$TIME_BETWEEN_FAILURES
|
||||
echo FAILURE_DURATION=$FAILURE_DURATION
|
||||
echo
|
||||
echo Press Ctrl+C to interrupt.
|
||||
echo
|
||||
|
||||
N=1
|
||||
|
||||
while :
|
||||
do
|
||||
|
||||
killNode $N
|
||||
|
||||
echo Waiting $FAILURE_DURATION s before attempting to reconnect mariadb_${N}
|
||||
sleep $FAILURE_DURATION
|
||||
|
||||
reconnectNode $N
|
||||
|
||||
echo Waiting $TIME_BETWEEN_FAILURES s before inducing another failure.
|
||||
echo
|
||||
sleep $TIME_BETWEEN_FAILURES
|
||||
|
||||
N=$((N+1))
|
||||
if [ "$N" -gt "$NODES" ]; then N=1; fi
|
||||
|
||||
done
|
7
testsuite/performance/db-failover/reconnect-node.sh
Executable file
7
testsuite/performance/db-failover/reconnect-node.sh
Executable file
|
@ -0,0 +1,7 @@
|
|||
#!/bin/bash
|
||||
|
||||
. ./common.sh
|
||||
|
||||
if [ -z $1 ]; then echo "Specify DB node to reconnect to cluster."; exit 1; fi
|
||||
|
||||
reconnectNode $1
|
38
testsuite/performance/db-failover/setup.sh
Executable file
38
testsuite/performance/db-failover/setup.sh
Executable file
|
@ -0,0 +1,38 @@
|
|||
#!/bin/bash
|
||||
|
||||
. ./common.sh
|
||||
|
||||
if [ -z "$DB_BOOTSTRAP_TIMEOUT" ]; then DB_BOOTSTRAP_TIMEOUT=10; fi
|
||||
if [ -z "$DB_JOIN_TIMEOUT" ]; then DB_JOIN_TIMEOUT=5; fi
|
||||
|
||||
echo Setting up Keycloak DB failover environment:
|
||||
|
||||
echo Starting DB bootstrap instance.
|
||||
docker-compose -f docker-compose-db-failover.yml up -d --build mariadb_bootstrap
|
||||
echo Waiting $DB_BOOTSTRAP_TIMEOUT s for the DB to initialize.
|
||||
sleep $DB_BOOTSTRAP_TIMEOUT
|
||||
|
||||
MARIADB_HOSTS=""
|
||||
for (( i=1; i<=$NODES; i++ )); do
|
||||
|
||||
MARIADB_HOSTS=$MARIADB_HOSTS,mariadb_$i:3306
|
||||
|
||||
echo Starting DB node $i.
|
||||
docker-compose -f docker-compose-db-failover.yml up -d mariadb_$i
|
||||
echo Waiting $DB_JOIN_TIMEOUT s for the DB node to join
|
||||
echo
|
||||
sleep $DB_JOIN_TIMEOUT
|
||||
|
||||
done
|
||||
|
||||
echo Turning off the DB bootstrap instance.
|
||||
docker-compose -f docker-compose-db-failover.yml stop mariadb_bootstrap
|
||||
|
||||
export MARIADB_HOSTS=${MARIADB_HOSTS/,/}
|
||||
echo MARIADB_HOSTS=$MARIADB_HOSTS
|
||||
|
||||
if $START_KEYCLOAK; then
|
||||
echo Starting Keycloak server.
|
||||
docker-compose -f docker-compose-db-failover.yml up -d --build keycloak
|
||||
./healthcheck.sh
|
||||
fi
|
8
testsuite/performance/db-failover/teardown.sh
Executable file
8
testsuite/performance/db-failover/teardown.sh
Executable file
|
@ -0,0 +1,8 @@
|
|||
#!/bin/bash
|
||||
|
||||
. ./common.sh
|
||||
|
||||
echo Stopping Keycloak DB failover environment.
|
||||
|
||||
docker-compose -f docker-compose-db-failover.yml down -v
|
||||
|
|
@ -3,7 +3,7 @@ general_log=OFF
|
|||
bind-address=0.0.0.0
|
||||
|
||||
innodb_flush_log_at_trx_commit=0
|
||||
query_cache_size=0
|
||||
query_cache_size=0
|
||||
query_cache_type=0
|
||||
|
||||
binlog_format=ROW
|
||||
|
@ -15,7 +15,12 @@ innodb_buffer_pool_size=122M
|
|||
|
||||
wsrep_on=ON
|
||||
wsrep_provider=/usr/lib/galera/libgalera_smm.so
|
||||
|
||||
wsrep_provider_options="gcache.size=300M; gcache.page_size=300M"
|
||||
#wsrep_provider_options="gcache.size=300M; gcache.page_size=300M; pc.bootstrap=YES"
|
||||
#wsrep_provider_options="gcache.size=300M; gcache.page_size=300M; pc.bootstrap=YES; pc.ignore_sb=TRUE"
|
||||
# See: http://galeracluster.com/documentation-webpages/twonode.html
|
||||
|
||||
wsrep_cluster_address="gcomm://"
|
||||
wsrep_cluster_name="galera_cluster_keycloak"
|
||||
wsrep_sst_method=rsync
|
||||
|
|
|
@ -41,7 +41,7 @@ services:
|
|||
CONFIGURATION: standalone-ha.xml
|
||||
PUBLIC_SUBNET: 10.0.1.0/24
|
||||
PRIVATE_SUBNET: 10.0.1.0/24
|
||||
MARIADB_HOST: mariadb
|
||||
MARIADB_HOSTS: mariadb:3306
|
||||
MARIADB_DATABASE: keycloak
|
||||
MARIADB_USER: keycloak
|
||||
MARIADB_PASSWORD: keycloak
|
||||
|
|
|
@ -131,7 +131,7 @@ services:
|
|||
CONFIGURATION: standalone-ha.xml
|
||||
PUBLIC_SUBNET: 10.1.1.0/24
|
||||
PRIVATE_SUBNET: 10.1.1.0/24
|
||||
MARIADB_HOST: mariadb_dc1
|
||||
MARIADB_HOSTS: mariadb_dc1:3306
|
||||
MARIADB_DATABASE: keycloak
|
||||
MARIADB_USER: keycloak
|
||||
MARIADB_PASSWORD: keycloak
|
||||
|
@ -172,7 +172,7 @@ services:
|
|||
CONFIGURATION: standalone-ha.xml
|
||||
PUBLIC_SUBNET: 10.2.1.0/24
|
||||
PRIVATE_SUBNET: 10.2.1.0/24
|
||||
MARIADB_HOST: mariadb_dc2
|
||||
MARIADB_HOSTS: mariadb_dc2:3306
|
||||
MARIADB_DATABASE: keycloak
|
||||
MARIADB_USER: keycloak
|
||||
MARIADB_PASSWORD: keycloak
|
||||
|
|
101
testsuite/performance/docker-compose-db-failover.yml
Normal file
101
testsuite/performance/docker-compose-db-failover.yml
Normal file
|
@ -0,0 +1,101 @@
|
|||
version: "2.2"
|
||||
|
||||
networks:
|
||||
keycloak:
|
||||
ipam:
|
||||
config:
|
||||
- subnet: 10.0.1.0/24
|
||||
|
||||
db_replication:
|
||||
ipam:
|
||||
config:
|
||||
- subnet: 10.0.3.0/24
|
||||
|
||||
services:
|
||||
|
||||
mariadb_bootstrap:
|
||||
build: db/mariadb
|
||||
image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest}
|
||||
networks:
|
||||
- db_replication
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: root
|
||||
MYSQL_INITDB_SKIP_TZINFO: foo
|
||||
MYSQL_DATABASE: keycloak
|
||||
MYSQL_USER: keycloak
|
||||
MYSQL_PASSWORD: keycloak
|
||||
entrypoint: docker-entrypoint-wsrep.sh
|
||||
command: --wsrep-new-cluster
|
||||
|
||||
mariadb_1:
|
||||
build: db/mariadb
|
||||
image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest}
|
||||
networks:
|
||||
- db_replication
|
||||
- keycloak
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: root
|
||||
MYSQL_INITDB_SKIP_TZINFO: foo
|
||||
entrypoint: docker-entrypoint-wsrep.sh
|
||||
command: --wsrep_cluster_address=gcomm://${MARIADB_RUNNING_HOST:-mariadb_bootstrap}
|
||||
ports:
|
||||
- "3316:3306"
|
||||
|
||||
mariadb_2:
|
||||
build: db/mariadb
|
||||
image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest}
|
||||
networks:
|
||||
- db_replication
|
||||
- keycloak
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: root
|
||||
MYSQL_INITDB_SKIP_TZINFO: foo
|
||||
entrypoint: docker-entrypoint-wsrep.sh
|
||||
command: --wsrep_cluster_address=gcomm://${MARIADB_RUNNING_HOST:-mariadb_bootstrap}
|
||||
ports:
|
||||
- "3326:3306"
|
||||
|
||||
mariadb_3:
|
||||
build: db/mariadb
|
||||
image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest}
|
||||
networks:
|
||||
- db_replication
|
||||
- keycloak
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: root
|
||||
MYSQL_INITDB_SKIP_TZINFO: foo
|
||||
entrypoint: docker-entrypoint-wsrep.sh
|
||||
command: --wsrep_cluster_address=gcomm://${MARIADB_RUNNING_HOST:-mariadb_bootstrap}
|
||||
ports:
|
||||
- "3336:3306"
|
||||
|
||||
|
||||
keycloak:
|
||||
build: keycloak
|
||||
image: keycloak_test_keycloak:${KEYCLOAK_VERSION:-latest}
|
||||
networks:
|
||||
- keycloak
|
||||
environment:
|
||||
MARIADB_HA_MODE: ${MARIADB_HA_MODE:-replication}
|
||||
MARIADB_HOSTS: ${MARIADB_HOSTS:-mariadb_1:3306,mariadb_2:3306}
|
||||
MARIADB_OPTIONS: ${MARIADB_OPTIONS}
|
||||
MARIADB_DATABASE: keycloak
|
||||
MARIADB_USER: keycloak
|
||||
MARIADB_PASSWORD: keycloak
|
||||
KEYCLOAK_USER: admin
|
||||
KEYCLOAK_PASSWORD: admin
|
||||
# docker-compose syntax note: ${ENV_VAR:-<DEFAULT_VALUE>}
|
||||
JAVA_OPTS: ${KEYCLOAK_JVM_MEMORY:--Xms64m -Xmx2g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true
|
||||
HTTP_MAX_CONNECTIONS: ${KEYCLOAK_HTTP_MAX_CONNECTIONS:-500}
|
||||
WORKER_IO_THREADS: ${KEYCLOAK_WORKER_IO_THREADS:-2}
|
||||
WORKER_TASK_MAX_THREADS: ${KEYCLOAK_WORKER_TASK_MAX_THREADS:-16}
|
||||
DS_MIN_POOL_SIZE: ${KEYCLOAK_DS_MIN_POOL_SIZE:-10}
|
||||
DS_MAX_POOL_SIZE: ${KEYCLOAK_DS_MAX_POOL_SIZE:-100}
|
||||
DS_POOL_PREFILL: "${KEYCLOAK_DS_POOL_PREFILL:-true}"
|
||||
DS_PS_CACHE_SIZE: ${KEYCLOAK_DS_PS_CACHE_SIZE:-100}
|
||||
ports:
|
||||
- "8080:8080"
|
||||
- "8443:8443"
|
||||
- "7989:7989"
|
||||
- "9990:9990"
|
||||
- "9999:9999"
|
|
@ -35,7 +35,7 @@ services:
|
|||
networks:
|
||||
- keycloak
|
||||
environment:
|
||||
MARIADB_HOST: mariadb
|
||||
MARIADB_HOSTS: mariadb:3306
|
||||
MARIADB_DATABASE: keycloak
|
||||
MARIADB_USER: keycloak
|
||||
MARIADB_PASSWORD: keycloak
|
||||
|
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
cd /subsystem=datasources/data-source=KeycloakDS
|
||||
|
||||
:write-attribute(name=connection-url, value=jdbc:mariadb://${env.MARIADB_HOST:mariadb}:${env.MARIADB_PORT:3306}/${env.MARIADB_DATABASE:keycloak})
|
||||
:write-attribute(name=connection-url, value=jdbc:mariadb:${env.MARIADB_HA_MODE:}://${env.MARIADB_HOSTS:mariadb:3306}/${env.MARIADB_DATABASE:keycloak}${env.MARIADB_OPTIONS:})
|
||||
:write-attribute(name=driver-name, value=mariadb)
|
||||
:write-attribute(name=user-name, value=${env.MARIADB_USER:keycloak})
|
||||
:write-attribute(name=password, value=${env.MARIADB_PASSWORD:keycloak})
|
||||
|
|
|
@ -34,6 +34,12 @@
|
|||
Uses maven-dependency-plugin to unpack keycloak-server-dist artifact into `target/keycloak` which is then added to Docker image.
|
||||
</description>
|
||||
|
||||
<properties>
|
||||
<server.groupId>org.keycloak</server.groupId>
|
||||
<server.artifactId>keycloak-server-dist</server.artifactId>
|
||||
<server.unpacked.folder>keycloak-${product.version}</server.unpacked.folder>
|
||||
</properties>
|
||||
|
||||
<build>
|
||||
|
||||
<plugins>
|
||||
|
@ -51,8 +57,8 @@
|
|||
<configuration>
|
||||
<artifactItems>
|
||||
<artifactItem>
|
||||
<groupId>org.keycloak</groupId>
|
||||
<artifactId>keycloak-server-dist</artifactId>
|
||||
<groupId>${server.groupId}</groupId>
|
||||
<artifactId>${server.artifactId}</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<type>zip</type>
|
||||
<outputDirectory>${project.build.directory}</outputDirectory>
|
||||
|
@ -77,7 +83,7 @@
|
|||
<configuration>
|
||||
<target>
|
||||
<move todir="${project.build.directory}/keycloak" failonerror="false">
|
||||
<fileset dir="${project.build.directory}/keycloak-${project.version}"/>
|
||||
<fileset dir="${project.build.directory}/${server.unpacked.folder}"/>
|
||||
</move>
|
||||
</target>
|
||||
</configuration>
|
||||
|
@ -87,4 +93,16 @@
|
|||
</plugins>
|
||||
</build>
|
||||
|
||||
<profiles>
|
||||
|
||||
<profile>
|
||||
<id>integration-testsuite-server</id>
|
||||
<properties>
|
||||
<server.groupId>org.keycloak.testsuite</server.groupId>
|
||||
<server.artifactId>integration-arquillian-servers-auth-server-wildfly</server.artifactId>
|
||||
<server.unpacked.folder>auth-server-wildfly</server.unpacked.folder>
|
||||
</properties>
|
||||
</profile>
|
||||
</profiles>
|
||||
|
||||
</project>
|
Loading…
Reference in a new issue