KEYCLOAK-1678 generalized cluster tests for any number of backend nodes
This commit is contained in:
Tomas Kyjovsky
parent
9fd9a1a5ad
commit
fa3c0bb0aa
4 changed files with 177 additions and 168 deletions
|
|
@ -1,19 +1,20 @@
|
|||
package org.keycloak.testsuite.cluster;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import org.jboss.arquillian.container.test.api.ContainerController;
|
||||
import org.jboss.arquillian.graphene.page.Page;
|
||||
import org.jboss.arquillian.test.api.ArquillianResource;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import org.junit.Before;
|
||||
import org.keycloak.admin.client.Keycloak;
|
||||
import org.keycloak.models.Constants;
|
||||
import org.keycloak.representations.idm.RealmRepresentation;
|
||||
import org.keycloak.testsuite.AbstractKeycloakTest;
|
||||
import org.keycloak.testsuite.arquillian.ContainerInfo;
|
||||
import org.keycloak.testsuite.auth.page.AuthRealm;
|
||||
import static org.keycloak.testsuite.auth.page.AuthRealm.ADMIN;
|
||||
import static org.keycloak.testsuite.auth.page.AuthRealm.MASTER;
|
||||
import static org.keycloak.testsuite.util.WaitUtils.pause;
|
||||
|
||||
/**
|
||||
*
|
||||
|
|
@ -24,52 +25,91 @@ public abstract class AbstractClusterTest extends AbstractKeycloakTest {
|
|||
@ArquillianResource
|
||||
protected ContainerController controller;
|
||||
|
||||
protected List<Keycloak> backendAdminClients = new ArrayList<>();
|
||||
protected Map<ContainerInfo, Keycloak> backendAdminClients = new HashMap<>();
|
||||
|
||||
public void startBackendNodes(int count) {
|
||||
if (count < 0 || count > 10) {
|
||||
throw new IllegalArgumentException();
|
||||
private int currentFailNodeIndex = 0;
|
||||
|
||||
public int getClusterSize() {
|
||||
return suiteContext.getAuthServerBackendsInfo().size();
|
||||
}
|
||||
|
||||
protected void iterateCurrentFailNode() {
|
||||
currentFailNodeIndex++;
|
||||
if (currentFailNodeIndex >= getClusterSize()) {
|
||||
currentFailNodeIndex = 0;
|
||||
}
|
||||
assertTrue(suiteContext.getAuthServerBackendsInfo().size() >= count);
|
||||
for (int i = 0; i < count; i++) {
|
||||
logCurrentState();
|
||||
}
|
||||
|
||||
ContainerInfo backendNode = suiteContext.getAuthServerBackendsInfo().get(i);
|
||||
protected ContainerInfo getCurrentFailNode() {
|
||||
return backendNode(currentFailNodeIndex);
|
||||
}
|
||||
|
||||
controller.start(backendNode.getQualifier());
|
||||
assertTrue(controller.isStarted(backendNode.getQualifier()));
|
||||
protected Set<ContainerInfo> getCurrentSurvivorNodes() {
|
||||
Set<ContainerInfo> survivors = new HashSet<>(suiteContext.getAuthServerBackendsInfo());
|
||||
survivors.remove(getCurrentFailNode());
|
||||
return survivors;
|
||||
}
|
||||
|
||||
backendAdminClients.add(createAdminClientFor(backendNode));
|
||||
protected void logCurrentState() {
|
||||
boolean started = controller.isStarted(getCurrentFailNode().getQualifier());
|
||||
log.info("Fail node: " + getCurrentFailNode() + (started ? "" : " (stopped)"));
|
||||
for (ContainerInfo survivor : getCurrentSurvivorNodes()) {
|
||||
started = controller.isStarted(survivor.getQualifier());
|
||||
log.info("Survivor: " + survivor + (started ? "" : " (stopped)"));
|
||||
}
|
||||
}
|
||||
|
||||
protected Keycloak createAdminClientFor(ContainerInfo backendNode) {
|
||||
log.info("Initializing admin client for " + backendNode.getContextRoot() + "/auth");
|
||||
return Keycloak.getInstance(backendNode.getContextRoot() + "/auth",
|
||||
MASTER, ADMIN, ADMIN, Constants.ADMIN_CLI_CLIENT_ID);
|
||||
public void failure() {
|
||||
log.info("Simulating failure");
|
||||
killBackendNode(getCurrentFailNode());
|
||||
}
|
||||
|
||||
public void failback() {
|
||||
log.info("Bringing all backend nodes online");
|
||||
for (ContainerInfo node : suiteContext.getAuthServerBackendsInfo()) {
|
||||
startBackendNode(node);
|
||||
}
|
||||
}
|
||||
|
||||
protected ContainerInfo backendNode(int i) {
|
||||
return suiteContext.getAuthServerBackendsInfo().get(i);
|
||||
}
|
||||
|
||||
protected void startBackendNode(int i) {
|
||||
String container = backendNode(i).getQualifier();
|
||||
if (!controller.isStarted(container)) {
|
||||
controller.start(container);
|
||||
backendAdminClients.set(i, createAdminClientFor(backendNode(i)));
|
||||
protected void startBackendNode(ContainerInfo node) {
|
||||
if (!controller.isStarted(node.getQualifier())) {
|
||||
log.info("Starting backend node: " + node);
|
||||
controller.start(node.getQualifier());
|
||||
assertTrue(controller.isStarted(node.getQualifier()));
|
||||
}
|
||||
log.info("Backend node " + node + " is started");
|
||||
if (!backendAdminClients.containsKey(node)) {
|
||||
backendAdminClients.put(node, createAdminClientFor(node));
|
||||
}
|
||||
}
|
||||
|
||||
protected void killBackendNode(int i) {
|
||||
backendAdminClients.get(i).close();
|
||||
controller.kill(backendNode(i).getQualifier());
|
||||
protected Keycloak createAdminClientFor(ContainerInfo node) {
|
||||
log.info("Initializing admin client for " + node.getContextRoot() + "/auth");
|
||||
return Keycloak.getInstance(node.getContextRoot() + "/auth",
|
||||
MASTER, ADMIN, ADMIN, Constants.ADMIN_CLI_CLIENT_ID);
|
||||
}
|
||||
|
||||
protected void listRealms(int i) {
|
||||
log.info(String.format("Node %s: AccessTokenString: %s", i + 1, backendAdminClients.get(i).tokenManager().getAccessTokenString()));
|
||||
for (RealmRepresentation r : backendAdminClients.get(i).realms().findAll()) {
|
||||
log.info(String.format("Node %s: Realm: %s, Id: %s", i + 1, r.getRealm(), r.getId()));
|
||||
}
|
||||
protected void killBackendNode(ContainerInfo node) {
|
||||
backendAdminClients.get(node).close();
|
||||
backendAdminClients.remove(node);
|
||||
log.info("Killing backend node: " + node);
|
||||
controller.kill(node.getQualifier());
|
||||
}
|
||||
|
||||
|
||||
protected Keycloak getAdminClientFor(ContainerInfo backendNode) {
|
||||
return backendAdminClients.get(backendNode);
|
||||
}
|
||||
|
||||
@Before
|
||||
public void beforeClusterTest() {
|
||||
failback();
|
||||
logCurrentState();
|
||||
pause(3000);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,57 +0,0 @@
|
|||
package org.keycloak.testsuite.cluster;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.keycloak.admin.client.Keycloak;
|
||||
import org.keycloak.testsuite.arquillian.ContainerInfo;
|
||||
import static org.keycloak.testsuite.util.WaitUtils.pause;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author tkyjovsk
|
||||
*/
|
||||
public abstract class AbstractTwoNodeClusterTest extends AbstractClusterTest {
|
||||
|
||||
@Before
|
||||
public void beforeTwoNodeClusterTest() {
|
||||
startBackendNodes(2);
|
||||
pause(3000);
|
||||
}
|
||||
|
||||
protected ContainerInfo backend1Info() {
|
||||
return backendNode(0);
|
||||
}
|
||||
|
||||
protected ContainerInfo backend2Info() {
|
||||
return backendNode(1);
|
||||
}
|
||||
|
||||
protected Keycloak backend1AdminClient() {
|
||||
return backendAdminClients.get(0);
|
||||
}
|
||||
|
||||
protected Keycloak backend2AdminClient() {
|
||||
return backendAdminClients.get(1);
|
||||
}
|
||||
|
||||
protected void startBackend1() {
|
||||
startBackendNode(0);
|
||||
}
|
||||
|
||||
protected void startBackend2() {
|
||||
startBackendNode(1);
|
||||
}
|
||||
|
||||
protected void failback() {
|
||||
startBackend1();
|
||||
startBackend2();
|
||||
}
|
||||
|
||||
protected void killBackend1() {
|
||||
killBackendNode(0);
|
||||
}
|
||||
|
||||
protected void killBackend2() {
|
||||
killBackendNode(1);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -5,81 +5,19 @@ import static org.junit.Assert.assertEquals;
|
|||
import static org.junit.Assert.assertFalse;
|
||||
import org.junit.Test;
|
||||
import org.keycloak.representations.idm.RealmRepresentation;
|
||||
import org.keycloak.testsuite.arquillian.ContainerInfo;
|
||||
import static org.keycloak.testsuite.auth.page.AuthRealm.TEST;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author tkyjovsk
|
||||
*/
|
||||
public class EntityInvalidationClusterTest extends AbstractTwoNodeClusterTest {
|
||||
public class EntityInvalidationClusterTest extends AbstractClusterTest {
|
||||
|
||||
@Override
|
||||
public void addTestRealms(List<RealmRepresentation> testRealms) {
|
||||
}
|
||||
|
||||
@Test
|
||||
public void realmCRUDWithoutFailover() {
|
||||
realmCRUD(TEST + "_wofo", false);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void realmCRUDWithFailover() {
|
||||
realmCRUD(TEST + "_wfo", true);
|
||||
}
|
||||
|
||||
public void realmCRUD(String realm, boolean containerFailover) {
|
||||
RealmRepresentation testRealm = new RealmRepresentation();
|
||||
testRealm.setRealm(realm);
|
||||
testRealm.setEnabled(true);
|
||||
|
||||
// CREATE on node1
|
||||
backend1AdminClient().realms().create(testRealm);
|
||||
|
||||
// check if created on node1
|
||||
RealmRepresentation testRealmOnBackend1 = backend1AdminClient().realms().realm(realm).toRepresentation();
|
||||
assertEquals(testRealmOnBackend1.getRealm(), testRealm.getRealm());
|
||||
if (containerFailover) {
|
||||
killBackend1();
|
||||
}
|
||||
|
||||
// check if created on node2
|
||||
RealmRepresentation testRealmOnBackend2 = backend2AdminClient().realms().realm(realm).toRepresentation();
|
||||
assertEquals(testRealmOnBackend1.getId(), testRealmOnBackend2.getId());
|
||||
assertEquals(testRealmOnBackend1.getRealm(), testRealmOnBackend2.getRealm());
|
||||
|
||||
failback();
|
||||
|
||||
// UPDATE on node2
|
||||
String realmUpdated = realm + "_updated";
|
||||
testRealmOnBackend2.setRealm(realmUpdated);
|
||||
backend2AdminClient().realms().realm(realm).update(testRealmOnBackend2);
|
||||
if (containerFailover) {
|
||||
killBackend2();
|
||||
}
|
||||
// check if updated on node1
|
||||
testRealmOnBackend1 = backend1AdminClient().realms().realm(realmUpdated).toRepresentation();
|
||||
assertEquals(testRealmOnBackend1.getId(), testRealmOnBackend2.getId());
|
||||
assertEquals(testRealmOnBackend1.getRealm(), testRealmOnBackend2.getRealm());
|
||||
|
||||
failback();
|
||||
|
||||
// DELETE on node1
|
||||
backend1AdminClient().realms().realm(realmUpdated).remove();
|
||||
if (containerFailover) {
|
||||
killBackend1();
|
||||
}
|
||||
// check if deleted on node2
|
||||
boolean testRealmOnBackend2Exists = false;
|
||||
for (RealmRepresentation realmOnBackend2 : backend2AdminClient().realms().findAll()) {
|
||||
if (realmUpdated.equals(realmOnBackend2.getRealm())
|
||||
|| testRealmOnBackend1.getId().equals(realmOnBackend2.getId())) {
|
||||
testRealmOnBackend2Exists = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
assertFalse(testRealmOnBackend2Exists);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void createRealmViaFrontend() {
|
||||
String realm = TEST + "_fe";
|
||||
|
|
@ -95,15 +33,102 @@ public class EntityInvalidationClusterTest extends AbstractTwoNodeClusterTest {
|
|||
RealmRepresentation testRealmOnFrontend = adminClient.realms().realm(realm).toRepresentation();
|
||||
assertEquals(testRealmOnFrontend.getRealm(), testRealm.getRealm());
|
||||
|
||||
// check if created on node1
|
||||
RealmRepresentation testRealmOnBackend1 = backend1AdminClient().realms().realm(realm).toRepresentation();
|
||||
assertEquals(testRealmOnBackend1.getId(), testRealmOnFrontend.getId());
|
||||
assertEquals(testRealmOnBackend1.getRealm(), testRealmOnFrontend.getRealm());
|
||||
|
||||
// check if created on node2
|
||||
RealmRepresentation testRealmOnBackend2 = backend2AdminClient().realms().realm(realm).toRepresentation();
|
||||
assertEquals(testRealmOnBackend2.getId(), testRealmOnFrontend.getId());
|
||||
assertEquals(testRealmOnBackend2.getRealm(), testRealmOnFrontend.getRealm());
|
||||
// check if created on backend nodes
|
||||
for (ContainerInfo backend : suiteContext.getAuthServerBackendsInfo()) {
|
||||
RealmRepresentation testRealmOnBackend = getAdminClientFor(backend).realms().realm(realm).toRepresentation();
|
||||
assertEquals(testRealmOnBackend.getId(), testRealmOnFrontend.getId());
|
||||
assertEquals(testRealmOnBackend.getRealm(), testRealmOnFrontend.getRealm());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void realmCRUDWithoutFailover() {
|
||||
realmCRUD(TEST + "_wofo", false);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void realmCRUDWithFailover() {
|
||||
realmCRUD(TEST + "_wfo", true);
|
||||
}
|
||||
|
||||
public void realmCRUD(String realmName, boolean backendFailover) {
|
||||
|
||||
// CREATE on current fail node
|
||||
log.info("Creating realm on : " + getCurrentFailNode());
|
||||
RealmRepresentation testRealm = new RealmRepresentation();
|
||||
testRealm.setRealm(realmName);
|
||||
testRealm.setEnabled(true);
|
||||
getAdminClientFor(getCurrentFailNode()).realms().create(testRealm);
|
||||
|
||||
// check if created on fail node
|
||||
RealmRepresentation testRealmOnFailNode = getAdminClientFor(getCurrentFailNode()).realms().realm(realmName).toRepresentation();
|
||||
assertEquals(testRealmOnFailNode.getRealm(), testRealm.getRealm());
|
||||
|
||||
if (backendFailover) {
|
||||
failure();
|
||||
}
|
||||
|
||||
// check if created on survivor nodes
|
||||
for (ContainerInfo survivorNode : getCurrentSurvivorNodes()) {
|
||||
RealmRepresentation testRealmOnSurvivorNode = getAdminClientFor(survivorNode).realms().realm(realmName).toRepresentation();
|
||||
assertEquals(testRealmOnFailNode.getId(), testRealmOnSurvivorNode.getId());
|
||||
assertEquals(testRealmOnFailNode.getRealm(), testRealmOnSurvivorNode.getRealm());
|
||||
log.info("Creation on survivor: " + survivorNode + " verified");
|
||||
}
|
||||
|
||||
failback();
|
||||
iterateCurrentFailNode();
|
||||
|
||||
// UPDATE on current fail node
|
||||
log.info("Updating realm on: " + getCurrentFailNode());
|
||||
String realmBeforeUpdate = realmName;
|
||||
realmName += "_updated";
|
||||
testRealm = testRealmOnFailNode;
|
||||
testRealm.setRealm(realmName);
|
||||
getAdminClientFor(getCurrentFailNode()).realms().realm(realmBeforeUpdate).update(testRealm);
|
||||
|
||||
// check if updated on fail node
|
||||
testRealmOnFailNode = getAdminClientFor(getCurrentFailNode()).realms().realm(realmName).toRepresentation();
|
||||
assertEquals(testRealmOnFailNode.getRealm(), testRealm.getRealm());
|
||||
|
||||
if (backendFailover) {
|
||||
failure();
|
||||
}
|
||||
|
||||
// check if updated on survivor nodes
|
||||
for (ContainerInfo survivorNode : getCurrentSurvivorNodes()) {
|
||||
RealmRepresentation testRealmOnSurvivorNode = getAdminClientFor(survivorNode).realms().realm(realmName).toRepresentation();
|
||||
assertEquals(testRealmOnFailNode.getId(), testRealmOnSurvivorNode.getId());
|
||||
assertEquals(testRealmOnFailNode.getRealm(), testRealmOnSurvivorNode.getRealm());
|
||||
log.info("Update on survivor: " + survivorNode + " verified");
|
||||
}
|
||||
|
||||
failback();
|
||||
iterateCurrentFailNode();
|
||||
|
||||
// DELETE on current fail node
|
||||
log.info("Deleting realm on: " + getCurrentFailNode());
|
||||
getAdminClientFor(getCurrentFailNode()).realms().realm(realmName).remove();
|
||||
|
||||
if (backendFailover) {
|
||||
failure();
|
||||
}
|
||||
|
||||
// check if deleted from all survivor nodes
|
||||
boolean realmStillExists = false;
|
||||
for (ContainerInfo survivorNode : getCurrentSurvivorNodes()) {
|
||||
boolean realmStillExistsOnSurvivor = false;
|
||||
for (RealmRepresentation realmOnSurvivorNode : getAdminClientFor(survivorNode).realms().findAll()) {
|
||||
if (realmName.equals(realmOnSurvivorNode.getRealm())
|
||||
|| testRealmOnFailNode.getId().equals(realmOnSurvivorNode.getId())) {
|
||||
realmStillExistsOnSurvivor = true;
|
||||
realmStillExists = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
log.error("Deletion on survivor: " + survivorNode + (realmStillExistsOnSurvivor ? " FAILED" : " verified"));
|
||||
}
|
||||
assertFalse(realmStillExists);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ import org.openqa.selenium.Cookie;
|
|||
*
|
||||
* @author tkyjovsk
|
||||
*/
|
||||
public class SessionFailoverClusterTest extends AbstractTwoNodeClusterTest {
|
||||
public class SessionFailoverClusterTest extends AbstractClusterTest {
|
||||
|
||||
public static final String KEYCLOAK_SESSION_COOKIE = "KEYCLOAK_SESSION";
|
||||
public static final String KEYCLOAK_IDENTITY_COOKIE = "KEYCLOAK_IDENTITY";
|
||||
|
|
@ -30,7 +30,7 @@ public class SessionFailoverClusterTest extends AbstractTwoNodeClusterTest {
|
|||
@Ignore("work in progress") // only works with owners="2" at the moment
|
||||
public void sessionFailover() {
|
||||
|
||||
// LOGOUT
|
||||
// LOGIN
|
||||
accountPage.navigateTo();
|
||||
driver.navigate().refresh();
|
||||
pause(3000);
|
||||
|
|
@ -40,7 +40,7 @@ public class SessionFailoverClusterTest extends AbstractTwoNodeClusterTest {
|
|||
Cookie sessionCookie = driver.manage().getCookieNamed(KEYCLOAK_SESSION_COOKIE);
|
||||
assertNotNull(sessionCookie);
|
||||
|
||||
killBackend1();
|
||||
failure();
|
||||
|
||||
// check if session survived backend failure
|
||||
|
||||
|
|
@ -53,6 +53,7 @@ public class SessionFailoverClusterTest extends AbstractTwoNodeClusterTest {
|
|||
assertEquals(sessionCookieAfterFailover.getValue(), sessionCookie.getValue());
|
||||
|
||||
failback();
|
||||
iterateCurrentFailNode();
|
||||
|
||||
// check if session survived backend failback
|
||||
driver.navigate().refresh();
|
||||
|
|
@ -71,7 +72,7 @@ public class SessionFailoverClusterTest extends AbstractTwoNodeClusterTest {
|
|||
sessionCookie = driver.manage().getCookieNamed(KEYCLOAK_SESSION_COOKIE);
|
||||
assertNull(sessionCookie);
|
||||
|
||||
killBackend1();
|
||||
failure();
|
||||
|
||||
// check if session survived backend failure
|
||||
driver.navigate().refresh();
|
||||
|
|
|
|||
Loading…
Reference in a new issue