From 805e9288fad3a7f0f690e44339335b4617a25927 Mon Sep 17 00:00:00 2001 From: Tomas Kyjovsky Date: Mon, 13 Nov 2017 23:38:01 +0100 Subject: [PATCH] KEYCLOAK-5588 Cross-dc: docker setup --- .../performance/docker-compose-crossdc.yml | 24 ++++++----- testsuite/performance/infinispan/Dockerfile | 9 +++- .../configs/add-keycloak-caches.cli | 15 ------- .../configs/add-keycloak-caches.cli.template | 32 ++++++++++++++ .../configs/add-private-network-interface.cli | 42 +++++++++++++++++++ ...-interface-for-jgroups-socket-bindings.cli | 9 ---- .../infinispan/docker-entrypoint-custom.sh | 2 +- testsuite/performance/keycloak/Dockerfile | 1 - .../configs/add-remote-cache-stores.cli | 22 +++++----- .../configs/distributed-cache-owners.cli | 14 +++++-- .../configs/infinispan-debug-logging.cli | 8 ++++ 11 files changed, 125 insertions(+), 53 deletions(-) delete mode 100644 testsuite/performance/infinispan/configs/add-keycloak-caches.cli create mode 100644 testsuite/performance/infinispan/configs/add-keycloak-caches.cli.template create mode 100644 testsuite/performance/infinispan/configs/add-private-network-interface.cli delete mode 100644 testsuite/performance/infinispan/configs/private-interface-for-jgroups-socket-bindings.cli create mode 100644 testsuite/performance/keycloak/configs/infinispan-debug-logging.cli diff --git a/testsuite/performance/docker-compose-crossdc.yml b/testsuite/performance/docker-compose-crossdc.yml index 4115efe67b..b20a6ebdca 100644 --- a/testsuite/performance/docker-compose-crossdc.yml +++ b/testsuite/performance/docker-compose-crossdc.yml @@ -30,8 +30,12 @@ networks: services: infinispan_dc1: - build: infinispan - image: keycloak_test_infinispan:${KEYCLOAK_VERSION:-latest} + build: + context: infinispan + args: + LOCAL_SITE: dc1 + REMOTE_SITE: dc2 + image: keycloak_test_infinispan_dc1:${KEYCLOAK_VERSION:-latest} cpus: 1 networks: - ispn_replication @@ -41,16 +45,18 @@ services: PRIVATE_SUBNET: 10.0.4.0/24 MGMT_USER: admin MGMT_USER_PASSWORD: admin -# APP_USER: keycloak -# APP_USER_PASSWORD: keycloak -# APP_USER_GROUPS: keycloak + TCP_PING_INITIAL_HOSTS: infinispan_dc1[7600] JAVA_OPTS: ${INFINISPAN_JVM_MEMORY:--Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -XX:+DisableExplicitGC} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true ports: - "9991:9990" infinispan_dc2: - build: infinispan - image: keycloak_test_infinispan:${KEYCLOAK_VERSION:-latest} + build: + context: infinispan + args: + LOCAL_SITE: dc2 + REMOTE_SITE: dc1 + image: keycloak_test_infinispan_dc2:${KEYCLOAK_VERSION:-latest} depends_on: infinispan_dc1: condition: service_healthy @@ -63,9 +69,7 @@ services: PRIVATE_SUBNET: 10.0.4.0/24 MGMT_USER: admin MGMT_USER_PASSWORD: admin -# APP_USER: keycloak -# APP_USER_PASSWORD: keycloak -# APP_USER_GROUPS: keycloak + TCP_PING_INITIAL_HOSTS: infinispan_dc1[7600],infinispan_dc2[7600] JAVA_OPTS: ${INFINISPAN_JVM_MEMORY:--Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -XX:+DisableExplicitGC} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true ports: - "9992:9990" diff --git a/testsuite/performance/infinispan/Dockerfile b/testsuite/performance/infinispan/Dockerfile index d6a4d81318..d151ac735b 100644 --- a/testsuite/performance/infinispan/Dockerfile +++ b/testsuite/performance/infinispan/Dockerfile @@ -1,6 +1,9 @@ FROM jboss/infinispan-server:8.2.6.Final #FROM jboss/infinispan-server:9.1.0.Final +ARG LOCAL_SITE +ARG REMOTE_SITE + USER root RUN yum -y install iproute USER jboss @@ -10,12 +13,14 @@ ENV CONFIGURATION clustered.xml ADD configs/ ./ ADD *.sh /usr/local/bin/ +RUN sed -e s/%LOCAL_SITE%/$LOCAL_SITE/ -e s/%REMOTE_SITE%/$REMOTE_SITE/ add-keycloak-caches.cli.template > add-keycloak-caches.cli + USER root RUN chmod -v +x /usr/local/bin/*.sh USER jboss -RUN $INFINISPAN_SERVER_HOME/bin/ispn-cli.sh --file=add-keycloak-caches.cli; \ - $INFINISPAN_SERVER_HOME/bin/ispn-cli.sh --file=private-interface-for-jgroups-socket-bindings.cli; \ +RUN $INFINISPAN_SERVER_HOME/bin/ispn-cli.sh --file=add-private-network-interface.cli; \ + $INFINISPAN_SERVER_HOME/bin/ispn-cli.sh --file=add-keycloak-caches.cli; \ cd $INFINISPAN_SERVER_HOME/standalone; rm -rf configuration/standalone_xml_history log data tmp HEALTHCHECK --interval=5s --timeout=5s --retries=12 CMD ["infinispan-healthcheck.sh"] diff --git a/testsuite/performance/infinispan/configs/add-keycloak-caches.cli b/testsuite/performance/infinispan/configs/add-keycloak-caches.cli deleted file mode 100644 index 317212cffc..0000000000 --- a/testsuite/performance/infinispan/configs/add-keycloak-caches.cli +++ /dev/null @@ -1,15 +0,0 @@ -embed-server --server-config=clustered.xml - -cd /subsystem=datagrid-infinispan/cache-container=clustered/configurations=CONFIGURATIONS - -#./replicated-cache-configuration=sessions-cfg:add(mode=SYNC, start=EAGER, batching=false) -./replicated-cache-configuration=sessions-cfg:add(mode=ASYNC, start=EAGER, batching=false) -./replicated-cache-configuration=sessions-cfg/transaction=TRANSACTION:add(locking=PESSIMISTIC, mode=NON_XA) - -cd /subsystem=datagrid-infinispan/cache-container=clustered - -./replicated-cache=work:add(configuration=sessions-cfg) -./replicated-cache=sessions:add(configuration=sessions-cfg) -./replicated-cache=offlineSessions:add(configuration=sessions-cfg) -./replicated-cache=actionTokens:add(configuration=sessions-cfg) -./replicated-cache=loginFailures:add(configuration=sessions-cfg) diff --git a/testsuite/performance/infinispan/configs/add-keycloak-caches.cli.template b/testsuite/performance/infinispan/configs/add-keycloak-caches.cli.template new file mode 100644 index 0000000000..2c1eeb4a5e --- /dev/null +++ b/testsuite/performance/infinispan/configs/add-keycloak-caches.cli.template @@ -0,0 +1,32 @@ +embed-server --server-config=clustered.xml + +# 2) +cd /subsystem=datagrid-jgroups + +# 2.a) +./channel=xsite:add(stack=tcp-private) + +# 2.b) +./stack=udp/relay=RELAY:add(site="%LOCAL_SITE%") +./stack=udp/relay=RELAY/remote-site=%REMOTE_SITE%:add(channel=xsite) + + + +# 3) +cd /subsystem=datagrid-infinispan/cache-container=clustered/configurations=CONFIGURATIONS + +./replicated-cache-configuration=sessions-cfg:add(mode=SYNC, start=EAGER, batching=false) +cd replicated-cache-configuration=sessions-cfg +./transaction=TRANSACTION:add(mode=NON_DURABLE_XA, locking=PESSIMISTIC) +./locking=LOCKING:add(acquire-timeout=0) +./backup=%REMOTE_SITE%:add(failure-policy=FAIL, strategy=SYNC, enabled=true, min-wait=60000, after-failures=3) + +cd /subsystem=datagrid-infinispan/cache-container=clustered +./replicated-cache=work:add(configuration=sessions-cfg) +./replicated-cache=sessions:add(configuration=sessions-cfg) +./replicated-cache=clientSessions:add(configuration=sessions-cfg) +./replicated-cache=offlineSessions:add(configuration=sessions-cfg) +./replicated-cache=offlineClientSessions:add(configuration=sessions-cfg) +./replicated-cache=actionTokens:add(configuration=sessions-cfg) +./replicated-cache=loginFailures:add(configuration=sessions-cfg) + diff --git a/testsuite/performance/infinispan/configs/add-private-network-interface.cli b/testsuite/performance/infinispan/configs/add-private-network-interface.cli new file mode 100644 index 0000000000..3cab3ebfaa --- /dev/null +++ b/testsuite/performance/infinispan/configs/add-private-network-interface.cli @@ -0,0 +1,42 @@ +embed-server --server-config=clustered.xml + +# ADD PRIVATE NETWORK INTERFATCE for cross-DC communication + +/interface=private:add(inet-address=${jboss.bind.address.private:127.0.0.1}) + + +# ADD PRIVATE JGROUPS SOCKET BINDINGS + +cd /socket-binding-group=standard-sockets + +./socket-binding=jgroups-mping-private:add( interface=private, port=0, multicast-address="${jboss.private.multicast.address:234.99.54.14}", multicast-port="45700") +./socket-binding=jgroups-tcp-private:add( interface=private, port=7600) +./socket-binding=jgroups-tcp-fd-private:add(interface=private, port=57600) +./socket-binding=jgroups-udp-private:add( interface=private, port=55200, multicast-address="${jboss.private.multicast.address:234.99.54.14}", multicast-port="45688") +./socket-binding=jgroups-udp-fd-private:add(interface=private, port=54200) + + +# ADD PRIVATE TCP STACK + +cd /subsystem=datagrid-jgroups + +./stack=tcp-private:add(transport={type=TCP, socket-binding=jgroups-tcp-private}) +cd stack=tcp-private + +:add-protocol(type=MPING, socket-binding=jgroups-mping-private) +:add-protocol(type=MERGE3) +:add-protocol(type=FD_SOCK, socket-binding=jgroups-tcp-fd-private) +:add-protocol(type=FD_ALL) +:add-protocol(type=VERIFY_SUSPECT) +:add-protocol(type=pbcast.NAKACK2, properties={"use_mcast_xmit" => "false"}) +:add-protocol(type=UNICAST3) +:add-protocol(type=pbcast.STABLE) +:add-protocol(type=pbcast.GMS) +:add-protocol(type=MFC) +# ISPN 8 +:add-protocol(type=FRAG2) +## ISPN 9 +#:add-protocol(type=FRAG3) +cd .. + + diff --git a/testsuite/performance/infinispan/configs/private-interface-for-jgroups-socket-bindings.cli b/testsuite/performance/infinispan/configs/private-interface-for-jgroups-socket-bindings.cli deleted file mode 100644 index ab38435dc1..0000000000 --- a/testsuite/performance/infinispan/configs/private-interface-for-jgroups-socket-bindings.cli +++ /dev/null @@ -1,9 +0,0 @@ -embed-server --server-config=clustered.xml - -/interface=private:add(inet-address=${jboss.bind.address.private:127.0.0.1}) - -/socket-binding-group=standard-sockets/socket-binding=jgroups-mping:write-attribute(name=interface, value=private) -/socket-binding-group=standard-sockets/socket-binding=jgroups-tcp:write-attribute(name=interface, value=private) -/socket-binding-group=standard-sockets/socket-binding=jgroups-tcp-fd:write-attribute(name=interface, value=private) -/socket-binding-group=standard-sockets/socket-binding=jgroups-udp:write-attribute(name=interface, value=private) -/socket-binding-group=standard-sockets/socket-binding=jgroups-udp-fd:write-attribute(name=interface, value=private) diff --git a/testsuite/performance/infinispan/docker-entrypoint-custom.sh b/testsuite/performance/infinispan/docker-entrypoint-custom.sh index 28f9bcd68e..75895847b5 100755 --- a/testsuite/performance/infinispan/docker-entrypoint-custom.sh +++ b/testsuite/performance/infinispan/docker-entrypoint-custom.sh @@ -4,7 +4,7 @@ cat $INFINISPAN_SERVER_HOME/standalone/configuration/$CONFIGURATION . get-ips.sh -PARAMS="-b $PUBLIC_IP -bmanagement $PUBLIC_IP -bprivate $PRIVATE_IP -Djgroups.bind_addr=$PRIVATE_IP -c $CONFIGURATION $@" +PARAMS="-b $PUBLIC_IP -bmanagement $PUBLIC_IP -bprivate $PRIVATE_IP -Djgroups.bind_addr=$PUBLIC_IP -c $CONFIGURATION $@" echo "Server startup params: $PARAMS" # Note: External container connectivity is always provided by eth0 -- irrespective of which is considered public/private by KC. diff --git a/testsuite/performance/keycloak/Dockerfile b/testsuite/performance/keycloak/Dockerfile index 91866e82c9..b89df12ef0 100644 --- a/testsuite/performance/keycloak/Dockerfile +++ b/testsuite/performance/keycloak/Dockerfile @@ -27,7 +27,6 @@ RUN curl --create-dirs --output modules/system/layers/base/org/mariadb/jdbc/main $JBOSS_HOME/bin/jboss-cli.sh --file=set-keycloak-ds.cli && \ $JBOSS_HOME/bin/jboss-cli.sh --file=io-worker-threads.cli && \ $JBOSS_HOME/bin/jboss-cli.sh --file=undertow.cli && \ - $JBOSS_HOME/bin/jboss-cli.sh --file=distributed-cache-owners.cli && \ $JBOSS_HOME/bin/jboss-cli.sh --file=modcluster-simple-load-provider.cli && \ if [ "$REMOTE_CACHES" == "true" ]; then $JBOSS_HOME/bin/jboss-cli.sh --file=add-remote-cache-stores.cli; fi && \ cd $JBOSS_HOME/standalone; rm -rf configuration/standalone_xml_history log data tmp ; \ diff --git a/testsuite/performance/keycloak/configs/add-remote-cache-stores.cli b/testsuite/performance/keycloak/configs/add-remote-cache-stores.cli index 6ebdff6ef3..e4b707ee8b 100644 --- a/testsuite/performance/keycloak/configs/add-remote-cache-stores.cli +++ b/testsuite/performance/keycloak/configs/add-remote-cache-stores.cli @@ -3,18 +3,18 @@ embed-server --server-config=standalone-ha.xml /subsystem=jgroups/stack=udp/transport=UDP:write-attribute(name=site, value=${env.SITE:dc1}) /socket-binding-group=standard-sockets/remote-destination-outbound-socket-binding=remote-cache:add(host=${env.INFINISPAN_HOST:localhost}, port=${env.INFINISPAN_PORT:11222}) +cd /subsystem=infinispan/cache-container=keycloak -/subsystem=infinispan/cache-container=keycloak:write-attribute(name=module, value=org.keycloak.keycloak-model-infinispan) +:write-attribute(name=module, value=org.keycloak.keycloak-model-infinispan) -/subsystem=infinispan/cache-container=keycloak/replicated-cache=work/store=remote:add(cache=work, fetch-state=false, passivation=false, preload=false, purge=false, remote-servers=["remote-cache"], shared=true) -/subsystem=infinispan/cache-container=keycloak/replicated-cache=work/store=remote:write-attribute(name=properties, value={rawValues=true, marshaller=org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory}) +./replicated-cache=work/store=remote:add(cache=work, fetch-state=false, passivation=false, preload=false, purge=false, remote-servers=["remote-cache"], shared=true, properties={rawValues=true, marshaller=org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory}) -/subsystem=infinispan/cache-container=keycloak/distributed-cache=sessions/store=custom:add(class=org.keycloak.models.sessions.infinispan.remotestore.KeycloakRemoteStoreConfigurationBuilder, fetch-state=false, passivation=false, preload=false, purge=false, shared=true) -/subsystem=infinispan/cache-container=keycloak/distributed-cache=sessions/store=custom:write-attribute(name=properties, value={remoteCacheName=sessions, useConfigTemplateFromCache=work}) - -/subsystem=infinispan/cache-container=keycloak/distributed-cache=offlineSessions/store=custom:add(class=org.keycloak.models.sessions.infinispan.remotestore.KeycloakRemoteStoreConfigurationBuilder, fetch-state=false, passivation=false, preload=false, purge=false, shared=true) -/subsystem=infinispan/cache-container=keycloak/distributed-cache=offlineSessions/store=custom:write-attribute(name=properties, value={remoteCacheName=offlineSessions, useConfigTemplateFromCache=work}) - -/subsystem=infinispan/cache-container=keycloak/distributed-cache=loginFailures/store=custom:add(class=org.keycloak.models.sessions.infinispan.remotestore.KeycloakRemoteStoreConfigurationBuilder, fetch-state=false, passivation=false, preload=false, purge=false, shared=true) -/subsystem=infinispan/cache-container=keycloak/distributed-cache=loginFailures/store=custom:write-attribute(name=properties, value={remoteCacheName=loginFailures, useConfigTemplateFromCache=work}) +./distributed-cache=sessions/store=remote:add(cache=sessions, fetch-state=false, passivation=false, preload=false, purge=false, remote-servers=["remote-cache"], shared=true, properties={rawValues=true, marshaller=org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory}) +./distributed-cache=offlineSessions/store=remote:add(cache=offlineSessions, fetch-state=false, passivation=false, preload=false, purge=false, remote-servers=["remote-cache"], shared=true, properties={rawValues=true, marshaller=org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory}) +./distributed-cache=clientSessions/store=remote:add(cache=clientSessions, fetch-state=false, passivation=false, preload=false, purge=false, remote-servers=["remote-cache"], shared=true, properties={rawValues=true, marshaller=org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory}) +./distributed-cache=offlineClientSessions/store=remote:add(cache=offlineClientSessions, fetch-state=false, passivation=false, preload=false, purge=false, remote-servers=["remote-cache"], shared=true, properties={rawValues=true, marshaller=org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory}) +./distributed-cache=loginFailures/store=remote:add(cache=loginFailures, fetch-state=false, passivation=false, preload=false, purge=false, remote-servers=["remote-cache"], shared=true, properties={rawValues=true, marshaller=org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory}) +./distributed-cache=actionTokens/store=remote:add(cache=actionTokens, fetch-state=false, passivation=false, preload=false, purge=false, remote-servers=["remote-cache"], shared=true, properties={rawValues=true, marshaller=org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory}) +./distributed-cache=actionTokens/eviction=EVICTION:add(max-entries=-1, strategy=NONE) +./distributed-cache=actionTokens/expiration=EXPIRATION:add(max-idle=-1,interval=300000) \ No newline at end of file diff --git a/testsuite/performance/keycloak/configs/distributed-cache-owners.cli b/testsuite/performance/keycloak/configs/distributed-cache-owners.cli index 5687e6d2c5..6618261c7b 100644 --- a/testsuite/performance/keycloak/configs/distributed-cache-owners.cli +++ b/testsuite/performance/keycloak/configs/distributed-cache-owners.cli @@ -1,7 +1,13 @@ embed-server --server-config=standalone-ha.xml # increase number of "owners" for distributed keycloak caches to support failover -/subsystem=infinispan/cache-container=keycloak/distributed-cache=sessions:write-attribute(name=owners, value=${distributed.cache.owners:2}) -/subsystem=infinispan/cache-container=keycloak/distributed-cache=authenticationSessions:write-attribute(name=owners, value=${distributed.cache.owners:2}) -/subsystem=infinispan/cache-container=keycloak/distributed-cache=offlineSessions:write-attribute(name=owners, value=${distributed.cache.owners:2}) -/subsystem=infinispan/cache-container=keycloak/distributed-cache=loginFailures:write-attribute(name=owners, value=${distributed.cache.owners:2}) + +cd /subsystem=infinispan/cache-container=keycloak/ + +./distributed-cache=sessions:write-attribute(name=owners, value=${distributed.cache.owners:2}) +./distributed-cache=offlineSessions:write-attribute(name=owners, value=${distributed.cache.owners:2}) +./distributed-cache=clientSessions:write-attribute(name=owners, value=${distributed.cache.owners:2}) +./distributed-cache=offlineClientSessions:write-attribute(name=owners, value=${distributed.cache.owners:2}) +./distributed-cache=loginFailures:write-attribute(name=owners, value=${distributed.cache.owners:2}) +./distributed-cache=actionTokens:write-attribute(name=owners, value=${distributed.cache.owners:2}) + diff --git a/testsuite/performance/keycloak/configs/infinispan-debug-logging.cli b/testsuite/performance/keycloak/configs/infinispan-debug-logging.cli new file mode 100644 index 0000000000..ed5d58f2b5 --- /dev/null +++ b/testsuite/performance/keycloak/configs/infinispan-debug-logging.cli @@ -0,0 +1,8 @@ +embed-server --server-config=standalone-ha.xml + +cd subsystem=logging + +./logger=org.keycloak.cluster.infinispan:add(level=DEBUG) +./logger=org.keycloak.connections.infinispan:add(level=DEBUG) +./logger=org.keycloak.models.cache.infinispan:add(level=DEBUG) +./logger=org.keycloak.models.sessions.infinispan:add(level=DEBUG)