libre.sh/keycloak-scim
3
0
Fork 0
Code Issues 15 Pull requests Releases Packages Activity Actions

KEYCLOAK-5716 KEYCLOAK-5738 Avoid infinispan deadlock. Ensure code-to-token works correctly in cross-dc

Browse source
This commit is contained in:
mposolda 2017-11-06 22:14:05 +01:00
parent 1db3134df8
commit 62a1c187a2
28 changed files with 206 additions and 108 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/Retry.java → common/src/main/java/org/keycloak/common/util/Retry.java
View file

@ -1,5 +1,5 @@
/*
* Copyright 2016 Red Hat, Inc. and/or its affiliates
* Copyright 2017 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -15,9 +15,7 @@
* limitations under the License.
*/
package org.keycloak.testsuite;
import java.util.function.Supplier;
package org.keycloak.common.util;
/**
* @author <a href="mailto:sthorger@redhat.com">Stian Thorgersen</a>
@ -44,7 +42,9 @@ public class Retry {
executionIndex++;
if (attemptsCount > 0) {
try {
if (intervalMillis > 0) {
Thread.sleep(intervalMillis);
}
} catch (InterruptedException ie) {
ie.addSuppressed(e);
throw new RuntimeException(ie);
@ -73,7 +73,9 @@ public class Retry {
attemptsCount--;
if (attemptsCount > 0) {
try {
if (intervalMillis > 0) {
Thread.sleep(intervalMillis);
}
} catch (InterruptedException ie) {
ie.addSuppressed(e);
throw new RuntimeException(ie);
@ -85,4 +87,19 @@ public class Retry {
}
}
/**
* Needed here just because java.util.function.Supplier defined from Java 8
*/
public interface Supplier<T> {
/**
* Gets a result.
*
* @return a result
*/
T get();
}
}

5
misc/CrossDataCenter.md
View file

@ -95,7 +95,8 @@ Infinispan Server setup
<cache-container name="clustered" default-cache="default" statistics="true">
...
<replicated-cache-configuration name="sessions-cfg" mode="SYNC" start="EAGER" batching="false">
<transaction mode="NON_XA" locking="PESSIMISTIC"/>
<transaction mode="NON_DURABLE_XA" locking="PESSIMISTIC"/>
<locking acquire-timeout="0" />
<backups>
<backup site="site2" failure-policy="FAIL" strategy="SYNC" enabled="true"/>
</backups>
@ -103,7 +104,9 @@ Infinispan Server setup
<replicated-cache name="work" configuration="sessions-cfg"/>
<replicated-cache name="sessions" configuration="sessions-cfg"/>
<replicated-cache name="clientSessions" configuration="sessions-cfg"/>
<replicated-cache name="offlineSessions" configuration="sessions-cfg"/>
<replicated-cache name="offlineClientSessions" configuration="sessions-cfg"/>
<replicated-cache name="actionTokens" configuration="sessions-cfg"/>
<replicated-cache name="loginFailures" configuration="sessions-cfg"/>

19
model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanCodeToTokenStoreProvider.java
View file

@ -22,6 +22,8 @@ import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
import org.infinispan.commons.api.BasicCache;
import org.jboss.logging.Logger;
import org.keycloak.common.util.Retry;
import org.keycloak.models.CodeToTokenStoreProvider;
import org.keycloak.models.KeycloakSession;
import org.keycloak.models.sessions.infinispan.entities.ActionTokenValueEntity;
@ -31,6 +33,8 @@ import org.keycloak.models.sessions.infinispan.entities.ActionTokenValueEntity;
*/
public class InfinispanCodeToTokenStoreProvider implements CodeToTokenStoreProvider {
public static final Logger logger = Logger.getLogger(InfinispanCodeToTokenStoreProvider.class);
private final Supplier<BasicCache<UUID, ActionTokenValueEntity>> codeCache;
private final KeycloakSession session;
@ -45,9 +49,24 @@ public class InfinispanCodeToTokenStoreProvider implements CodeToTokenStoreProvi
int lifespanInSeconds = session.getContext().getRealm().getAccessCodeLifespan();
boolean codeAlreadyExists = Retry.call(() -> {
try {
BasicCache<UUID, ActionTokenValueEntity> cache = codeCache.get();
ActionTokenValueEntity existing = cache.putIfAbsent(codeId, tokenValue, lifespanInSeconds, TimeUnit.SECONDS);
return existing == null;
} catch (RuntimeException re) {
if (logger.isDebugEnabled()) {
logger.debugf(re, "Failed when adding code %s", codeId);
}
// Rethrow the exception. Retry will take care of handle the exception and eventually retry the operation.
throw re;
}
}, 3, 0);
return codeAlreadyExists;
}
@Override

4
model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProviderFactory.java
View file

@ -255,6 +255,10 @@ public class InfinispanUserSessionProviderFactory implements UserSessionProvider
RemoteCache<K, SessionEntityWrapper<V>> remoteCache = (RemoteCache) remoteStores.iterator().next().getRemoteCache();
if (remoteCache == null) {
throw new IllegalStateException("No remote cache available for the infinispan cache: " + ispnCache.getName());
}
remoteCacheInvoker.addRemoteCache(ispnCache.getName(), remoteCache, maxIdleLoader);
RemoteCacheSessionListener hotrodListener = RemoteCacheSessionListener.createListener(session, ispnCache, remoteCache);

20
model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/RemoteCacheInvoker.java
View file

@ -17,6 +17,7 @@
package org.keycloak.models.sessions.infinispan.remotestore;
import org.keycloak.common.util.Retry;
import org.keycloak.common.util.Time;
import java.util.Collections;
import java.util.HashMap;
@ -32,6 +33,7 @@ import org.keycloak.models.KeycloakSession;
import org.keycloak.models.RealmModel;
import org.keycloak.models.sessions.infinispan.changes.SessionEntityWrapper;
import org.keycloak.models.sessions.infinispan.changes.SessionUpdateTask;
import org.keycloak.models.sessions.infinispan.entities.LoginFailureEntity;
import org.keycloak.models.sessions.infinispan.entities.SessionEntity;
import org.keycloak.models.sessions.infinispan.entities.UserSessionEntity;
@ -71,14 +73,28 @@ public class RemoteCacheInvoker {
return;
}
long maxIdleTimeMs = context.maxIdleTimeLoader.getMaxIdleTimeMs(realm);
long loadedMaxIdleTimeMs = context.maxIdleTimeLoader.getMaxIdleTimeMs(realm);
// Double the timeout to ensure that entry won't expire on remoteCache in case that write of some entities to remoteCache is postponed (eg. userSession.lastSessionRefresh)
maxIdleTimeMs = maxIdleTimeMs * 2;
final long maxIdleTimeMs = loadedMaxIdleTimeMs * 2;
logger.debugf("Running task '%s' on remote cache '%s' . Key is '%s'", operation, cacheName, key);
Retry.execute(() -> {
try {
runOnRemoteCache(context.remoteCache, maxIdleTimeMs, key, task, sessionWrapper);
} catch (RuntimeException re) {
if (logger.isDebugEnabled()) {
logger.debugf(re, "Failed running task '%s' on remote cache '%s' . Key: '%s' . Will try to retry the task",
operation, cacheName, key);
}
// Rethrow the exception. Retry will take care of handle the exception and eventually retry the operation.
throw re;
}
}, 10, 0);
}

36
model/infinispan/src/test/java/org/keycloak/cluster/infinispan/ConcurrencyJDGRemoteCacheTest.java
View file

@ -30,6 +30,7 @@ import org.infinispan.client.hotrod.annotation.ClientCacheEntryModified;
import org.infinispan.client.hotrod.annotation.ClientListener;
import org.infinispan.client.hotrod.event.ClientCacheEntryCreatedEvent;
import org.infinispan.client.hotrod.event.ClientCacheEntryModifiedEvent;
import org.infinispan.client.hotrod.exceptions.HotRodClientException;
import org.infinispan.manager.EmbeddedCacheManager;
import org.infinispan.persistence.manager.PersistenceManager;
import org.infinispan.persistence.remote.RemoteStore;
@ -47,9 +48,13 @@ public class ConcurrencyJDGRemoteCacheTest {
private static Map<String, EntryInfo> state = new HashMap<>();
private RemoteCache remoteCache1;
private RemoteCache remoteCache2;
public static void main(String[] args) throws Exception {
// Init map somehow
for (int i=0 ; i<30 ; i++) {
for (int i=0 ; i<3000 ; i++) {
String key = "key-" + i;
state.put(key, new EntryInfo());
}
@ -58,6 +63,8 @@ public class ConcurrencyJDGRemoteCacheTest {
Worker worker1 = createWorker(1);
Worker worker2 = createWorker(2);
long start = System.currentTimeMillis();
// Start and join workers
worker1.start();
worker2.start();
@ -65,12 +72,16 @@ public class ConcurrencyJDGRemoteCacheTest {
worker1.join();
worker2.join();
long took = System.currentTimeMillis() - start;
// Output
for (Map.Entry<String, EntryInfo> entry : state.entrySet()) {
System.out.println(entry.getKey() + ":::" + entry.getValue());
worker1.cache.remove(entry.getKey());
}
System.out.println("Took: " + took + " ms");
// Finish JVM
worker1.cache.getCacheManager().stop();
worker2.cache.getCacheManager().stop();
@ -127,7 +138,7 @@ public class ConcurrencyJDGRemoteCacheTest {
String cacheKey = entry.getKey();
EntryInfo wrapper = state.get(cacheKey);
int val = getClusterStartupTime(this.cache, cacheKey, wrapper);
int val = getClusterStartupTime(this.cache, cacheKey, wrapper, myThreadId);
if (myThreadId == 1) {
wrapper.th1.set(val);
} else {
@ -141,8 +152,8 @@ public class ConcurrencyJDGRemoteCacheTest {
}
public static int getClusterStartupTime(Cache<String, Integer> cache, String cacheKey, EntryInfo wrapper) {
Integer startupTime = new Random().nextInt(1024);
public static int getClusterStartupTime(Cache<String, Integer> cache, String cacheKey, EntryInfo wrapper, int myThreadId) {
Integer startupTime = myThreadId==1 ? Integer.parseInt(cacheKey.substring(4)) : Integer.parseInt(cacheKey.substring(4)) * 2;
// Concurrency doesn't work correctly with this
//Integer existingClusterStartTime = (Integer) cache.putIfAbsent(cacheKey, startupTime);
@ -154,21 +165,25 @@ public class ConcurrencyJDGRemoteCacheTest {
for (int i=0 ; i<10 ; i++) {
try {
existingClusterStartTime = (Integer) remoteCache.withFlags(Flag.FORCE_RETURN_VALUE).putIfAbsent(cacheKey, startupTime);
} catch (Exception ce) {
break;
} catch (HotRodClientException ce) {
if (i == 9) {
throw ce;
//break;
} else {
System.err.println("EXception: i=" + i);
wrapper.exceptions.incrementAndGet();
System.err.println("Exception: i=" + i + " for key: " + cacheKey + " and myThreadId: " + myThreadId);
}
}
}
if (existingClusterStartTime == null || startupTime.equals(remoteCache.get(cacheKey))) {
if (existingClusterStartTime == null
// || startupTime.equals(remoteCache.get(cacheKey))
) {
wrapper.successfulInitializations.incrementAndGet();
return startupTime;
} else {
System.err.println("Not equal!!! startupTime=" + startupTime + ", existingClusterStartTime=" + existingClusterStartTime );
wrapper.failedInitializations.incrementAndGet();
return existingClusterStartTime;
}
}
@ -178,10 +193,13 @@ public class ConcurrencyJDGRemoteCacheTest {
AtomicInteger successfulListenerWrites = new AtomicInteger(0);
AtomicInteger th1 = new AtomicInteger();
AtomicInteger th2 = new AtomicInteger();
AtomicInteger failedInitializations = new AtomicInteger();
AtomicInteger exceptions = new AtomicInteger();
@Override
public String toString() {
return String.format("Inits: %d, listeners: %d, th1: %d, th2: %d", successfulInitializations.get(), successfulListenerWrites.get(), th1.get(), th2.get());
return String.format("Inits: %d, listeners: %d, failedInits: %d, exceptions: %s, th1: %d, th2: %d", successfulInitializations.get(), successfulListenerWrites.get(),
failedInitializations.get(), exceptions.get(), th1.get(), th2.get());
}
}

4
model/infinispan/src/test/java/org/keycloak/cluster/infinispan/ConcurrencyJDGRemoveSessionTest.java
View file

@ -121,6 +121,8 @@ public class ConcurrencyJDGRemoveSessionTest {
logger.info("SESSIONS NOT AVAILABLE ON DC2");
long took = System.currentTimeMillis() - start;
logger.infof("took %d ms", took);
// // Start and join workers
// worker1.start();
@ -137,8 +139,6 @@ public class ConcurrencyJDGRemoveSessionTest {
cache2.getCacheManager().stop();
}
long took = System.currentTimeMillis() - start;
// // Output
// for (Map.Entry<String, EntryInfo> entry : state.entrySet()) {
// System.out.println(entry.getKey() + ":::" + entry.getValue());

30
model/infinispan/src/test/java/org/keycloak/cluster/infinispan/ConcurrencyJDGSessionsCacheTest.java
View file

@ -187,6 +187,10 @@ public class ConcurrencyJDGSessionsCacheTest {
", successfulListenerWrites: " + successfulListenerWrites.get() + ", successfulListenerWrites2: " + successfulListenerWrites2.get() +
", failedReplaceCounter: " + failedReplaceCounter.get() + ", failedReplaceCounter2: " + failedReplaceCounter2.get());
System.out.println("remoteCache1.notes: " + ((UserSessionEntity) remoteCache1.get("123")).getNotes().size() );
System.out.println("remoteCache2.notes: " + ((UserSessionEntity) remoteCache2.get("123")).getNotes().size() );
System.out.println("Histogram: ");
//histogram.dumpStats();
@ -314,14 +318,26 @@ public class ConcurrencyJDGSessionsCacheTest {
// In case it's hardcoded (eg. all the replaces are doing same change, so session is defacto not changed), then histogram may contain bigger value than 1 on some places.
//String noteKey = "some";
boolean replaced = false;
while (!replaced) {
ReplaceStatus replaced = ReplaceStatus.NOT_REPLACED;
while (replaced != ReplaceStatus.REPLACED) {
VersionedValue<UserSessionEntity> versioned = remoteCache.getVersioned("123");
UserSessionEntity oldSession = versioned.getValue();
//UserSessionEntity clone = DistributedCacheConcurrentWritesTest.cloneSession(oldSession);
UserSessionEntity clone = oldSession;
// In case that exception was thrown (ReplaceStatus.ERROR), the remoteCache may have the note. Seems that transactions are not fully rolled-back on the JDG side
// in case that backup fails
if (replaced == ReplaceStatus.NOT_REPLACED) {
clone.getNotes().put(noteKey, "someVal");
} else if (replaced == ReplaceStatus.ERROR) {
if (clone.getNotes().containsKey(noteKey)) {
System.err.println("I HAVE THE KEY: " + noteKey);
} else {
System.err.println("I DON'T HAVE THE KEY: " + noteKey);
clone.getNotes().put(noteKey, "someVal");
}
}
//cache.replace("123", clone);
replaced = cacheReplace(versioned, clone);
}
@ -336,7 +352,7 @@ public class ConcurrencyJDGSessionsCacheTest {
}
private boolean cacheReplace(VersionedValue<UserSessionEntity> oldSession, UserSessionEntity newSession) {
private ReplaceStatus cacheReplace(VersionedValue<UserSessionEntity> oldSession, UserSessionEntity newSession) {
try {
boolean replaced = remoteCache.replaceWithVersion("123", newSession, oldSession.getVersion());
//boolean replaced = true;
@ -348,15 +364,19 @@ public class ConcurrencyJDGSessionsCacheTest {
} else {
histogram.increaseSuccessOpsCount(oldSession.getVersion());
}
return replaced;
return replaced ? ReplaceStatus.REPLACED : ReplaceStatus.NOT_REPLACED;
} catch (Exception re) {
failedReplaceCounter2.incrementAndGet();
return false;
return ReplaceStatus.ERROR;
}
//return replaced;
}
}
private enum ReplaceStatus {
REPLACED, NOT_REPLACED, ERROR
}
/*
// Worker, which operates on "classic" cache and rely on operations delegated to the second cache
private static class CacheWorker extends Thread {

3
testsuite/integration-arquillian/servers/cache-server/jboss/common/add-keycloak-caches.xsl
View file

@ -36,7 +36,8 @@
<xsl:apply-templates select="@* | node()" />
<replicated-cache-configuration name="sessions-cfg" mode="SYNC" start="EAGER" batching="false">
<transaction mode="NON_XA" locking="PESSIMISTIC"/>
<transaction mode="NON_DURABLE_XA" locking="PESSIMISTIC"/>
<locking acquire-timeout="0" />
<backups>
<backup site="{$remote.site}" failure-policy="FAIL" strategy="SYNC" enabled="true"/>
</backups>

2
testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/CacheStatisticsControllerEnricher.java
View file

@ -35,7 +35,7 @@ import org.jboss.arquillian.core.spi.Validate;
import org.jboss.arquillian.test.spi.TestEnricher;
import org.jboss.logging.Logger;
import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
import org.keycloak.testsuite.Retry;
import org.keycloak.common.util.Retry;
import org.keycloak.testsuite.arquillian.annotation.JmxInfinispanCacheStatistics;
import org.keycloak.testsuite.arquillian.annotation.JmxInfinispanChannelStatistics;
import org.keycloak.testsuite.arquillian.jmx.JmxConnectorRegistry;

4
testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/AbstractKeycloakTest.java
View file

@ -433,6 +433,10 @@ public abstract class AbstractKeycloakTest {
}
}
public Logger getLogger() {
return log;
}
private static void enableHTTPSForAuthServer() throws IOException, CommandFailedException, TimeoutException, InterruptedException, CliException, OperationException {
OnlineManagementClient client = AuthServerTestEnricher.getManagementClient();
Administration administration = new Administration(client);

2
testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/adapter/servlet/cluster/AbstractSAMLAdapterClusterTest.java
View file

@ -18,7 +18,7 @@ package org.keycloak.testsuite.adapter.servlet.cluster;
import org.keycloak.admin.client.resource.RealmResource;
import org.keycloak.representations.idm.*;
import org.keycloak.testsuite.Retry;
import org.keycloak.common.util.Retry;
import org.keycloak.testsuite.adapter.page.EmployeeServletDistributable;
import org.keycloak.testsuite.adapter.page.SAMLServlet;
import org.keycloak.testsuite.auth.page.AuthRealm;

14
testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/admin/concurrency/AbstractConcurrencyTest.java
View file

@ -17,9 +17,11 @@
package org.keycloak.testsuite.admin.concurrency;
import org.jboss.logging.Logger;
import org.keycloak.admin.client.Keycloak;
import org.keycloak.admin.client.resource.RealmResource;
import org.keycloak.representations.idm.RealmRepresentation;
import org.keycloak.testsuite.AbstractKeycloakTest;
import org.keycloak.testsuite.AbstractTestRealmKeycloakTest;
import java.util.LinkedList;
import java.util.Collection;
@ -54,6 +56,11 @@ public abstract class AbstractConcurrencyTest extends AbstractTestRealmKeycloakT
}
protected void run(final int numThreads, final int totalNumberOfExecutions, final KeycloakRunnable... runnables) {
run(numThreads, totalNumberOfExecutions, this, runnables);
}
public static void run(final int numThreads, final int totalNumberOfExecutions, AbstractKeycloakTest testImpl, final KeycloakRunnable... runnables) {
final ExecutorService service = SYNCHRONIZED
? Executors.newSingleThreadExecutor()
: Executors.newFixedThreadPool(numThreads);
@ -61,7 +68,7 @@ public abstract class AbstractConcurrencyTest extends AbstractTestRealmKeycloakT
ThreadLocal<Keycloak> keycloaks = new ThreadLocal<Keycloak>() {
@Override
protected Keycloak initialValue() {
return Keycloak.getInstance(getAuthServerRoot().toString(), "master", "admin", "admin", org.keycloak.models.Constants.ADMIN_CLI_CLIENT_ID);
return Keycloak.getInstance(testImpl.getAuthServerRoot().toString(), "master", "admin", "admin", org.keycloak.models.Constants.ADMIN_CLI_CLIENT_ID);
}
};
@ -95,12 +102,13 @@ public abstract class AbstractConcurrencyTest extends AbstractTestRealmKeycloakT
if (! failures.isEmpty()) {
RuntimeException ex = new RuntimeException("There were failures in threads. Failures count: " + failures.size());
failures.forEach(ex::addSuppressed);
failures.forEach(e -> log.error(e.getMessage(), e));
failures.forEach(e -> testImpl.getLogger().error(e.getMessage(), e));
throw ex;
}
}
protected interface KeycloakRunnable {
public interface KeycloakRunnable {
void run(int threadIndex, Keycloak keycloak, RealmResource realm) throws Throwable;

8
testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/admin/concurrency/ConcurrentLoginTest.java
View file

@ -50,7 +50,7 @@ import org.keycloak.admin.client.resource.ClientsResource;
import org.keycloak.admin.client.resource.RealmResource;
import org.keycloak.representations.AccessToken;
import org.keycloak.representations.idm.ClientRepresentation;
import org.keycloak.testsuite.Retry;
import org.keycloak.common.util.Retry;
import org.keycloak.testsuite.admin.ApiUtil;
import org.keycloak.testsuite.util.ClientBuilder;
import org.keycloak.testsuite.util.OAuthClient;
@ -222,9 +222,9 @@ public class ConcurrentLoginTest extends AbstractConcurrencyTest {
oauth1.openLogout();
// Code should be successfully exchanged for the token just once
Assert.assertEquals(1, codeToTokenSuccessCount.get());
Assert.assertEquals(DEFAULT_THREADS - 1, codeToTokenErrorsCount.get());
// Code should be successfully exchanged for the token at max once. In some cases (EG. Cross-DC) it may not be even successfully exchanged
Assert.assertThat(codeToTokenSuccessCount.get(), Matchers.lessThanOrEqualTo(1));
Assert.assertThat(codeToTokenErrorsCount.get(), Matchers.greaterThanOrEqualTo(DEFAULT_THREADS - 1));
log.infof("Iteration %d passed successfully", i);
}

2
testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/broker/AbstractBaseBrokerTest.java
View file

@ -26,7 +26,7 @@ import org.keycloak.representations.idm.RealmRepresentation;
import org.keycloak.representations.idm.UserRepresentation;
import org.keycloak.testsuite.AbstractKeycloakTest;
import org.keycloak.testsuite.Assert;
import org.keycloak.testsuite.Retry;
import org.keycloak.common.util.Retry;
import org.keycloak.testsuite.admin.ApiUtil;
import org.keycloak.testsuite.pages.AccountPasswordPage;
import org.keycloak.testsuite.pages.AccountUpdateProfilePage;

2
testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractAdminCrossDCTest.java
View file

@ -19,7 +19,7 @@ package org.keycloak.testsuite.crossdc;
import org.keycloak.admin.client.resource.RealmResource;
import org.keycloak.events.log.JBossLoggingEventListenerProviderFactory;
import org.keycloak.representations.idm.RealmRepresentation;
import org.keycloak.testsuite.Retry;
import org.keycloak.common.util.Retry;
import org.keycloak.testsuite.arquillian.InfinispanStatistics;
import org.keycloak.testsuite.events.EventsListenerProviderFactory;
import org.keycloak.testsuite.util.TestCleanup;

2
testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/ActionTokenCrossDCTest.java
View file

@ -20,7 +20,7 @@ import org.keycloak.admin.client.resource.UserResource;
import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
import org.keycloak.models.UserModel;
import org.keycloak.representations.idm.UserRepresentation;
import org.keycloak.testsuite.Retry;
import org.keycloak.common.util.Retry;
import org.keycloak.testsuite.admin.ApiUtil;
import org.keycloak.testsuite.page.LoginPasswordUpdatePage;
import org.keycloak.testsuite.pages.ErrorPage;

45
testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/BruteForceCrossDCTest.java
View file

@ -21,12 +21,16 @@ import java.io.IOException;
import java.net.URISyntaxException;
import javax.ws.rs.NotFoundException;
import org.hamcrest.Matchers;
import org.jboss.arquillian.container.test.api.Deployment;
import org.jboss.arquillian.container.test.api.TargetsContainer;
import org.jboss.shrinkwrap.api.spec.WebArchive;
import org.junit.Before;
import org.junit.Test;
import org.keycloak.admin.client.Keycloak;
import org.keycloak.admin.client.resource.RealmResource;
import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
import org.keycloak.models.Constants;
import org.keycloak.models.RealmModel;
@ -36,7 +40,8 @@ import org.keycloak.representations.idm.RealmRepresentation;
import org.keycloak.representations.idm.UserRepresentation;
import org.keycloak.testsuite.AbstractTestRealmKeycloakTest;
import org.keycloak.testsuite.Assert;
import org.keycloak.testsuite.Retry;
import org.keycloak.common.util.Retry;
import org.keycloak.testsuite.admin.concurrency.AbstractConcurrencyTest;
import org.keycloak.testsuite.client.KeycloakTestingClient;
import org.keycloak.testsuite.runonserver.RunOnServerDeployment;
import org.keycloak.testsuite.util.ClientBuilder;
@ -210,6 +215,44 @@ public class BruteForceCrossDCTest extends AbstractAdminCrossDCTest {
}
@Test
public void testBruteForceConcurrentUpdate() throws Exception {
// Enable 1st node on each DC only
enableDcOnLoadBalancer(DC.FIRST);
enableDcOnLoadBalancer(DC.SECOND);
// Clear all
adminClient.realms().realm(REALM_NAME).attackDetection().clearAllBruteForce();
assertStatistics("After brute force cleared", 0, 0, 0);
// create the entry manually in DC0
addUserLoginFailure(getTestingClientForStartedNodeInDc(0));
assertStatistics("After create entry1", 1, 0, 1);
AbstractConcurrencyTest.KeycloakRunnable runnable = new AbstractConcurrencyTest.KeycloakRunnable() {
@Override
public void run(int threadIndex, Keycloak keycloak, RealmResource realm) throws Throwable {
createBruteForceFailures(1, "login-test-1");
}
};
AbstractConcurrencyTest.run(2, 20, this, runnable);
Retry.execute(() -> {
int dc0user1 = (Integer) getAdminClientForStartedNodeInDc(0).realm(REALM_NAME).attackDetection().bruteForceUserStatus("login-test-1").get("numFailures");
int dc1user1 = (Integer) getAdminClientForStartedNodeInDc(1).realm(REALM_NAME).attackDetection().bruteForceUserStatus("login-test-1").get("numFailures");
log.infof("After concurrent update entry1: dc0User1=%d, dc1user1=%d", dc0user1, dc1user1);
// The numFailures can be actually bigger than 20. Conflicts can increase the numFailures number to bigger value as they may not be fully reverted (listeners etc)
Assert.assertThat(dc0user1, Matchers.greaterThan(20));
Assert.assertEquals(dc0user1, dc1user1);
}, 50, 50);
}
private void assertStatistics(String prefixMessage, int expectedUser1, int expectedUser2, int expectedCacheSize) {
Retry.execute(() -> {
int dc0user1 = (Integer) getAdminClientForStartedNodeInDc(0).realm(REALM_NAME).attackDetection().bruteForceUserStatus("login-test-1").get("numFailures");

10
testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/ConcurrentLoginCrossDCTest.java
View file

@ -17,6 +17,7 @@
package org.keycloak.testsuite.crossdc;
import org.junit.Assert;
import org.keycloak.admin.client.Keycloak;
import org.keycloak.admin.client.resource.RealmResource;
import java.util.List;
@ -67,15 +68,6 @@ public class ConcurrentLoginCrossDCTest extends ConcurrentLoginTest {
}
// TODO: We know that this test won't work in cross-dc setup based on "backup "caches. But we need to add the test that clientSessions
// are invalidated after attempt of reuse the same code multiple times
@Test
@Override
@Ignore
public void concurrentCodeReuseShouldFail() throws Throwable {
}
@Test
public void concurrentLoginWithRandomDcFailures() throws Throwable {
log.info("*********************************************");

2
testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/InvalidationCrossDCTest.java
View file

@ -31,7 +31,7 @@ import org.keycloak.representations.idm.RealmRepresentation;
import org.keycloak.representations.idm.UserRepresentation;
import org.keycloak.representations.idm.authorization.ResourceRepresentation;
import org.keycloak.testsuite.Assert;
import org.keycloak.testsuite.Retry;
import org.keycloak.common.util.Retry;
import org.keycloak.testsuite.admin.ApiUtil;
/**

2
testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/LastSessionRefreshCrossDCTest.java
View file

@ -24,7 +24,7 @@ import org.junit.Test;
import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
import org.keycloak.representations.idm.RealmRepresentation;
import org.keycloak.testsuite.Assert;
import org.keycloak.testsuite.Retry;
import org.keycloak.common.util.Retry;
import org.keycloak.testsuite.arquillian.ContainerInfo;
import org.keycloak.testsuite.rest.representation.RemoteCacheStats;
import org.keycloak.testsuite.util.OAuthClient;

1
testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/LoginCrossDCTest.java
View file

@ -22,7 +22,6 @@ import javax.ws.rs.core.Response;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.junit.Test;
import org.keycloak.testsuite.Assert;
import org.keycloak.testsuite.Retry;
import org.keycloak.testsuite.util.Matchers;
import org.keycloak.testsuite.util.OAuthClient;

2
testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/SessionExpirationCrossDCTest.java
View file

@ -36,7 +36,7 @@ import org.keycloak.representations.idm.RealmRepresentation;
import org.keycloak.representations.idm.UserRepresentation;
import org.keycloak.representations.idm.UserSessionRepresentation;
import org.keycloak.testsuite.Assert;
import org.keycloak.testsuite.Retry;
import org.keycloak.common.util.Retry;
import org.keycloak.testsuite.admin.ApiUtil;
import org.keycloak.testsuite.arquillian.InfinispanStatistics;
import org.keycloak.testsuite.arquillian.annotation.JmxInfinispanCacheStatistics;

2
testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/forms/LogoutTest.java
View file

@ -25,7 +25,7 @@ import org.keycloak.representations.idm.UserRepresentation;
import org.keycloak.testsuite.Assert;
import org.keycloak.testsuite.AssertEvents;
import org.keycloak.testsuite.AbstractTestRealmKeycloakTest;
import org.keycloak.testsuite.Retry;
import org.keycloak.common.util.Retry;
import org.keycloak.testsuite.admin.ApiUtil;
import org.keycloak.testsuite.pages.AppPage;
import org.keycloak.testsuite.pages.LoginPage;

2
testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/session/LastSessionRefreshUnitTest.java
View file

@ -37,7 +37,7 @@ import org.keycloak.models.sessions.infinispan.changes.sessions.SessionData;
import org.keycloak.models.sessions.infinispan.entities.UserSessionEntity;
import org.keycloak.representations.idm.RealmRepresentation;
import org.keycloak.testsuite.AbstractKeycloakTest;
import org.keycloak.testsuite.Retry;
import org.keycloak.common.util.Retry;
import org.keycloak.testsuite.runonserver.RunOnServer;
import org.keycloak.testsuite.runonserver.RunOnServerDeployment;

45
testsuite/integration-deprecated/src/test/java/org/keycloak/testsuite/Retry.java
View file

@ -1,45 +0,0 @@
/*
* Copyright 2016 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.testsuite;
/**
* @author <a href="mailto:sthorger@redhat.com">Stian Thorgersen</a>
*/
public class Retry {
public static void execute(Runnable runnable, int retry, long interval) {
while (true) {
try {
runnable.run();
return;
} catch (RuntimeException e) {
retry--;
if (retry > 0) {
try {
Thread.sleep(interval);
} catch (InterruptedException ie) {
throw new RuntimeException(ie);
}
} else {
throw e;
}
}
}
}
}

1
testsuite/integration-deprecated/src/test/java/org/keycloak/testsuite/keycloaksaml/SamlAdapterTestStrategy.java
View file

@ -45,7 +45,6 @@ import org.keycloak.saml.common.constants.*;
import org.keycloak.saml.processing.core.saml.v2.constants.X500SAMLProfileConstants;
import org.keycloak.services.managers.RealmManager;
import org.keycloak.testsuite.KeycloakServer;
import org.keycloak.testsuite.Retry;
import org.keycloak.testsuite.pages.LoginPage;
import org.keycloak.testsuite.rule.AbstractKeycloakRule;
import org.keycloak.testsuite.rule.ErrorServlet;

2
testsuite/integration-deprecated/src/test/java/org/keycloak/testsuite/rule/AbstractKeycloakRule.java
View file

@ -31,6 +31,7 @@ import org.junit.rules.TemporaryFolder;
import org.keycloak.Config;
import org.keycloak.adapters.KeycloakConfigResolver;
import org.keycloak.adapters.servlet.KeycloakOIDCFilter;
import org.keycloak.common.util.Retry;
import org.keycloak.common.util.Time;
import org.keycloak.models.KeycloakSession;
import org.keycloak.models.KeycloakTransaction;
@ -41,7 +42,6 @@ import org.keycloak.representations.idm.RealmRepresentation;
import org.keycloak.representations.idm.UserRepresentation;
import org.keycloak.services.managers.RealmManager;
import org.keycloak.testsuite.KeycloakServer;
import org.keycloak.testsuite.Retry;
import org.keycloak.util.JsonSerialization;
import javax.servlet.DispatcherType;