libre.sh/keycloak-scim
3
0
Fork 0
Code Issues 15 Pull requests Releases Packages Activity Actions

KEYCLOAK-5691 Galera cluster, full testsuite

Browse source
This commit is contained in:
Tomas Kyjovsky 2017-10-17 17:36:11 +02:00
parent 9e2ab2750a
commit a45a2acc4c
17 changed files with 356 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
testsuite/integration-arquillian/servers/auth-server/jboss/pom.xml
View file

@ -855,5 +855,23 @@
</modules>
</profile>
<profile>
<id>db-failover-mariadb</id>
<properties>
<jdbc.mvn.groupId>org.mariadb.jdbc</jdbc.mvn.groupId>
<jdbc.mvn.artifactId>mariadb-java-client</jdbc.mvn.artifactId>
<jdbc.mvn.version>2.0.3</jdbc.mvn.version>
<keycloak.connectionsJpa.user>keycloak</keycloak.connectionsJpa.user>
<keycloak.connectionsJpa.password>keycloak</keycloak.connectionsJpa.password>
<mariadb.ha.mode>replication</mariadb.ha.mode>
<mariadb.hosts>localhost:3316,localhost:3326</mariadb.hosts>
<mariadb.database>keycloak</mariadb.database>
<mariadb.options></mariadb.options>
<keycloak.connectionsJpa.url>jdbc:mariadb:${mariadb.ha.mode}://${mariadb.hosts}/${mariadb.database}${mariadb.options}</keycloak.connectionsJpa.url>
</properties>
</profile>
</profiles>
</project>

67
testsuite/performance/db-failover/README.md Normal file
View file

@ -0,0 +1,67 @@
# DB Failover Testing Utilities
A set of scripts for testing DB failover scenarios.
The scripts expect to be run with relative execution prefix `./` from within the `db-failover` directory.
Provisioned services are defined in `../docker-compose-db-failover.yml` template.
## Set the size of DB cluster
Default size is 2 nodes. For a 3-node cluster run: `export NODES=3` before executing any of the scripts.
For more than 3 nodes more service definitions need to be added to the docker-compose template.
## Set up the environment
Run `./setup.sh`
This script will:
1. Start a bootstrap DB instance of MariaDB cluster
2. Start additional DB instances connected to the bootstrapped cluster
3. Stop the bootstrap DB instance
4. Optionally start Keycloak server
Parameterized by environment variables:
- `MARIADB_HA_MODE` See: [MariaDB HA parameters](https://mariadb.com/kb/en/library/failover-and-high-availability-with-mariadb-connector-j/#failover-high-availability-parameters)
Defaults to `replication`.
- `MARIADB_OPTIONS` See: [MariaDB HA options](https://mariadb.com/kb/en/library/failover-and-high-availability-with-mariadb-connector-j/#failover-high-availability-options).
Use format: `?option1=value1[&option2=value2]...`. Default is an empty string.
- `START_KEYCLOAK` Default is `false`. Use `export START_KEYCLOAK=true` to enable.
More options relevant to MariaDB clustering can be found in `../db/mariadb/wsrep.cnf`.
## Test the failover
### Manual failover
To induce a failure of specific DB node run: `./kill-node.sh X` where `X ∈ {1..3}`
To reconnect the node back run: `./reconnect-node.sh X`
### Automated failover loop
Run `./loop.sh`
This script will run an infinite loop of failover/failback of DB nodes, switching to the next node in each loop.
Parameterized by environment variables:
- `TIME_BETWEEN_FAILURES` Default is `60` (seconds).
- `FAILURE_DURATION` Default is `60` (seconds).
To exit the script press `Ctrl+C`.
### Check number of table rows across the cluster
Run: `./check-rows.sh`
## Tear down the environment
Run `./teardown.sh`
This will stop all services and delete the database.

22
testsuite/performance/db-failover/check-rows.sh Executable file
View file

@ -0,0 +1,22 @@
#!/bin/bash
. ./common.sh
CONCAT_SQL=$(< db-failover/concat.sql)
CONCAT_SQL_COMMAND='mysql -N -B -u keycloak --password=keycloak -e "$CONCAT_SQL" keycloak'
ROWS_SQL=$(eval docker-compose -f docker-compose-db-failover.yml exec mariadb_1 $CONCAT_SQL_COMMAND | tr -dc '[:print:]')
ROWS_SQL=${ROWS_SQL%UNION }
ROWS_SQL_COMMAND='mysql -u keycloak --password=keycloak -e "$ROWS_SQL" keycloak'
for (( i=1; i <= $NODES; i++)); do
ROWS[i]=$(eval docker-compose -f docker-compose-db-failover.yml exec mariadb_$i $ROWS_SQL_COMMAND)
done
DIFF=0
for (( i=2; i <= $NODES; i++)); do
echo Node 1 vs Node $(( i )):
diff -y --suppress-common-lines <(echo "${ROWS[1]}") <(echo "${ROWS[i]}")
if [ $? -eq 0 ]; then echo No difference.; else DIFF=1; fi
done
exit $DIFF

18
testsuite/performance/db-failover/common.sh Executable file
View file

@ -0,0 +1,18 @@
function killNode {
echo Killing mariadb_${1}
docker-compose -f docker-compose-db-failover.yml kill mariadb_${1}
}
function reconnectNode {
N=$1
NR=$(( N + 1 )); if [ "$NR" -gt "$NODES" ]; then NR=1; fi
export MARIADB_RUNNING_HOST=mariadb_${NR}
echo Attempting failback of mariadb_${N}, connecting to running cluster member mariadb_${NR}
docker-compose -f docker-compose-db-failover.yml up -d mariadb_${N}
}
if [ -z $NODES ]; then export NODES=2; fi
if [ -z $MARIADB_OPTIONS ]; then export MARIADB_OPTIONS=""; fi
if [ -z $START_KEYCLOAK ]; then export START_KEYCLOAK=false; fi
cd ..

3
testsuite/performance/db-failover/concat.sql Normal file
View file

@ -0,0 +1,3 @@
SELECT CONCAT(
'SELECT "', table_name, '" AS table_name, COUNT(*) AS exact_row_count FROM `', table_schema, '`.`', table_name, '` UNION '
) FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = 'keycloak';

7
testsuite/performance/db-failover/kill-node.sh Executable file
View file

@ -0,0 +1,7 @@
#!/bin/bash
. ./common.sh
if [ -z $1 ]; then echo "Specify DB node to kill."; exit 1; fi
killNode $1

35
testsuite/performance/db-failover/loop.sh Executable file
View file

@ -0,0 +1,35 @@
#!/bin/bash
. ./common.sh
if [ -z "$TIME_BETWEEN_FAILURES" ]; then export TIME_BETWEEN_FAILURES=60; fi
if [ -z "$FAILURE_DURATION" ]; then export FAILURE_DURATION=60; fi
echo Running DB failover loop with the following parameters:
echo NODES=$NODES
echo TIME_BETWEEN_FAILURES=$TIME_BETWEEN_FAILURES
echo FAILURE_DURATION=$FAILURE_DURATION
echo
echo Press Ctrl+C to interrupt.
echo
N=1
while :
do
killNode $N
echo Waiting $FAILURE_DURATION s before attempting to reconnect mariadb_${N}
sleep $FAILURE_DURATION
reconnectNode $N
echo Waiting $TIME_BETWEEN_FAILURES s before inducing another failure.
echo
sleep $TIME_BETWEEN_FAILURES
N=$((N+1))
if [ "$N" -gt "$NODES" ]; then N=1; fi
done

7
testsuite/performance/db-failover/reconnect-node.sh Executable file
View file

@ -0,0 +1,7 @@
#!/bin/bash
. ./common.sh
if [ -z $1 ]; then echo "Specify DB node to reconnect to cluster."; exit 1; fi
reconnectNode $1

38
testsuite/performance/db-failover/setup.sh Executable file
View file

@ -0,0 +1,38 @@
#!/bin/bash
. ./common.sh
if [ -z "$DB_BOOTSTRAP_TIMEOUT" ]; then DB_BOOTSTRAP_TIMEOUT=10; fi
if [ -z "$DB_JOIN_TIMEOUT" ]; then DB_JOIN_TIMEOUT=5; fi
echo Setting up Keycloak DB failover environment:
echo Starting DB bootstrap instance.
docker-compose -f docker-compose-db-failover.yml up -d --build mariadb_bootstrap
echo Waiting $DB_BOOTSTRAP_TIMEOUT s for the DB to initialize.
sleep $DB_BOOTSTRAP_TIMEOUT
MARIADB_HOSTS=""
for (( i=1; i<=$NODES; i++ )); do
MARIADB_HOSTS=$MARIADB_HOSTS,mariadb_$i:3306
echo Starting DB node $i.
docker-compose -f docker-compose-db-failover.yml up -d mariadb_$i
echo Waiting $DB_JOIN_TIMEOUT s for the DB node to join
echo
sleep $DB_JOIN_TIMEOUT
done
echo Turning off the DB bootstrap instance.
docker-compose -f docker-compose-db-failover.yml stop mariadb_bootstrap
export MARIADB_HOSTS=${MARIADB_HOSTS/,/}
echo MARIADB_HOSTS=$MARIADB_HOSTS
if $START_KEYCLOAK; then
echo Starting Keycloak server.
docker-compose -f docker-compose-db-failover.yml up -d --build keycloak
./healthcheck.sh
fi

8
testsuite/performance/db-failover/teardown.sh Executable file
View file

@ -0,0 +1,8 @@
#!/bin/bash
. ./common.sh
echo Stopping Keycloak DB failover environment.
docker-compose -f docker-compose-db-failover.yml down -v

7
testsuite/performance/db/mariadb/wsrep.cnf
View file

@ -3,7 +3,7 @@ general_log=OFF
bind-address=0.0.0.0
innodb_flush_log_at_trx_commit=0
query_cache_size=0
query_cache_size=0
query_cache_type=0
binlog_format=ROW
@ -15,7 +15,12 @@ innodb_buffer_pool_size=122M
wsrep_on=ON
wsrep_provider=/usr/lib/galera/libgalera_smm.so
wsrep_provider_options="gcache.size=300M; gcache.page_size=300M"
#wsrep_provider_options="gcache.size=300M; gcache.page_size=300M; pc.bootstrap=YES"
#wsrep_provider_options="gcache.size=300M; gcache.page_size=300M; pc.bootstrap=YES; pc.ignore_sb=TRUE"
# See: http://galeracluster.com/documentation-webpages/twonode.html
wsrep_cluster_address="gcomm://"
wsrep_cluster_name="galera_cluster_keycloak"
wsrep_sst_method=rsync

2
testsuite/performance/docker-compose-cluster.yml
View file

@ -41,7 +41,7 @@ services:
CONFIGURATION: standalone-ha.xml
PUBLIC_SUBNET: 10.0.1.0/24
PRIVATE_SUBNET: 10.0.1.0/24
MARIADB_HOST: mariadb
MARIADB_HOSTS: mariadb:3306
MARIADB_DATABASE: keycloak
MARIADB_USER: keycloak
MARIADB_PASSWORD: keycloak

4
testsuite/performance/docker-compose-crossdc.yml
View file

@ -131,7 +131,7 @@ services:
CONFIGURATION: standalone-ha.xml
PUBLIC_SUBNET: 10.1.1.0/24
PRIVATE_SUBNET: 10.1.1.0/24
MARIADB_HOST: mariadb_dc1
MARIADB_HOSTS: mariadb_dc1:3306
MARIADB_DATABASE: keycloak
MARIADB_USER: keycloak
MARIADB_PASSWORD: keycloak
@ -172,7 +172,7 @@ services:
CONFIGURATION: standalone-ha.xml
PUBLIC_SUBNET: 10.2.1.0/24
PRIVATE_SUBNET: 10.2.1.0/24
MARIADB_HOST: mariadb_dc2
MARIADB_HOSTS: mariadb_dc2:3306
MARIADB_DATABASE: keycloak
MARIADB_USER: keycloak
MARIADB_PASSWORD: keycloak

101
testsuite/performance/docker-compose-db-failover.yml Normal file
View file

@ -0,0 +1,101 @@
version: "2.2"
networks:
keycloak:
ipam:
config:
- subnet: 10.0.1.0/24
db_replication:
ipam:
config:
- subnet: 10.0.3.0/24
services:
mariadb_bootstrap:
build: db/mariadb
image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest}
networks:
- db_replication
environment:
MYSQL_ROOT_PASSWORD: root
MYSQL_INITDB_SKIP_TZINFO: foo
MYSQL_DATABASE: keycloak
MYSQL_USER: keycloak
MYSQL_PASSWORD: keycloak
entrypoint: docker-entrypoint-wsrep.sh
command: --wsrep-new-cluster
mariadb_1:
build: db/mariadb
image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest}
networks:
- db_replication
- keycloak
environment:
MYSQL_ROOT_PASSWORD: root
MYSQL_INITDB_SKIP_TZINFO: foo
entrypoint: docker-entrypoint-wsrep.sh
command: --wsrep_cluster_address=gcomm://${MARIADB_RUNNING_HOST:-mariadb_bootstrap}
ports:
- "3316:3306"
mariadb_2:
build: db/mariadb
image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest}
networks:
- db_replication
- keycloak
environment:
MYSQL_ROOT_PASSWORD: root
MYSQL_INITDB_SKIP_TZINFO: foo
entrypoint: docker-entrypoint-wsrep.sh
command: --wsrep_cluster_address=gcomm://${MARIADB_RUNNING_HOST:-mariadb_bootstrap}
ports:
- "3326:3306"
mariadb_3:
build: db/mariadb
image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest}
networks:
- db_replication
- keycloak
environment:
MYSQL_ROOT_PASSWORD: root
MYSQL_INITDB_SKIP_TZINFO: foo
entrypoint: docker-entrypoint-wsrep.sh
command: --wsrep_cluster_address=gcomm://${MARIADB_RUNNING_HOST:-mariadb_bootstrap}
ports:
- "3336:3306"
keycloak:
build: keycloak
image: keycloak_test_keycloak:${KEYCLOAK_VERSION:-latest}
networks:
- keycloak
environment:
MARIADB_HA_MODE: ${MARIADB_HA_MODE:-replication}
MARIADB_HOSTS: ${MARIADB_HOSTS:-mariadb_1:3306,mariadb_2:3306}
MARIADB_OPTIONS: ${MARIADB_OPTIONS}
MARIADB_DATABASE: keycloak
MARIADB_USER: keycloak
MARIADB_PASSWORD: keycloak
KEYCLOAK_USER: admin
KEYCLOAK_PASSWORD: admin
# docker-compose syntax note: ${ENV_VAR:-<DEFAULT_VALUE>}
JAVA_OPTS: ${KEYCLOAK_JVM_MEMORY:--Xms64m -Xmx2g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true
HTTP_MAX_CONNECTIONS: ${KEYCLOAK_HTTP_MAX_CONNECTIONS:-500}
WORKER_IO_THREADS: ${KEYCLOAK_WORKER_IO_THREADS:-2}
WORKER_TASK_MAX_THREADS: ${KEYCLOAK_WORKER_TASK_MAX_THREADS:-16}
DS_MIN_POOL_SIZE: ${KEYCLOAK_DS_MIN_POOL_SIZE:-10}
DS_MAX_POOL_SIZE: ${KEYCLOAK_DS_MAX_POOL_SIZE:-100}
DS_POOL_PREFILL: "${KEYCLOAK_DS_POOL_PREFILL:-true}"
DS_PS_CACHE_SIZE: ${KEYCLOAK_DS_PS_CACHE_SIZE:-100}
ports:
- "8080:8080"
- "8443:8443"
- "7989:7989"
- "9990:9990"
- "9999:9999"

2
testsuite/performance/docker-compose.yml
View file

@ -35,7 +35,7 @@ services:
networks:
- keycloak
environment:
MARIADB_HOST: mariadb
MARIADB_HOSTS: mariadb:3306
MARIADB_DATABASE: keycloak
MARIADB_USER: keycloak
MARIADB_PASSWORD: keycloak

2
testsuite/performance/keycloak/configs/set-keycloak-ds-batch.cli
View file

@ -2,7 +2,7 @@
cd /subsystem=datasources/data-source=KeycloakDS
:write-attribute(name=connection-url, value=jdbc:mariadb://${env.MARIADB_HOST:mariadb}:${env.MARIADB_PORT:3306}/${env.MARIADB_DATABASE:keycloak})
:write-attribute(name=connection-url, value=jdbc:mariadb:${env.MARIADB_HA_MODE:}://${env.MARIADB_HOSTS:mariadb:3306}/${env.MARIADB_DATABASE:keycloak}${env.MARIADB_OPTIONS:})
:write-attribute(name=driver-name, value=mariadb)
:write-attribute(name=user-name, value=${env.MARIADB_USER:keycloak})
:write-attribute(name=password, value=${env.MARIADB_PASSWORD:keycloak})

24
testsuite/performance/keycloak/pom.xml
View file

@ -34,6 +34,12 @@
Uses maven-dependency-plugin to unpack keycloak-server-dist artifact into `target/keycloak` which is then added to Docker image.
</description>
<properties>
<server.groupId>org.keycloak</server.groupId>
<server.artifactId>keycloak-server-dist</server.artifactId>
<server.unpacked.folder>keycloak-${product.version}</server.unpacked.folder>
</properties>
<build>
<plugins>
@ -51,8 +57,8 @@
<configuration>
<artifactItems>
<artifactItem>
<groupId>org.keycloak</groupId>
<artifactId>keycloak-server-dist</artifactId>
<groupId>${server.groupId}</groupId>
<artifactId>${server.artifactId}</artifactId>
<version>${project.version}</version>
<type>zip</type>
<outputDirectory>${project.build.directory}</outputDirectory>
@ -77,7 +83,7 @@
<configuration>
<target>
<move todir="${project.build.directory}/keycloak" failonerror="false">
<fileset dir="${project.build.directory}/keycloak-${project.version}"/>
<fileset dir="${project.build.directory}/${server.unpacked.folder}"/>
</move>
</target>
</configuration>
@ -87,4 +93,16 @@
</plugins>
</build>
<profiles>
<profile>
<id>integration-testsuite-server</id>
<properties>
<server.groupId>org.keycloak.testsuite</server.groupId>
<server.artifactId>integration-arquillian-servers-auth-server-wildfly</server.artifactId>
<server.unpacked.folder>auth-server-wildfly</server.unpacked.folder>
</properties>
</profile>
</profiles>
</project>