libre.sh/keycloak-scim
2
0
Fork 0
Code Issues 14 Pull requests Releases Packages Activity Actions 6

KEYCLOAK-2495 Improve startup time with many offlineSessions in UserSessionPersister

Browse source
This commit is contained in:
mposolda 2016-02-17 10:59:48 +01:00
parent 1328531f31
commit f52f998bcd
7 changed files with 100 additions and 60 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

57
misc/Testsuite.md
View file

@ -143,6 +143,61 @@ kinit hnelson@KEYCLOAK.ORG
and provide password `secret`
Now when you access `http://localhost:8081/auth/realms/master/account` you should be logged in automatically as user `hnelson` .
Now when you access `http://localhost:8081/auth/realms/master/account` you should be logged in automatically as user `hnelson` .
Create many users or offline sessions
-------------------------------------
Run testsuite with the command like this:
```
mvn exec:java -Pkeycloak-server -DstartTestsuiteCLI
```
Alternatively if you want to use your MySQL database use the command like this (replace properties values according your DB connection):
```
mvn exec:java -Pkeycloak-server -Dkeycloak.connectionsJpa.url=jdbc:mysql://localhost/keycloak -Dkeycloak.connectionsJpa.driver=com.mysql.jdbc.Driver -Dkeycloak.connectionsJpa.user=keycloak -Dkeycloak.connectionsJpa.password=keycloak -DstartTestsuiteCLI
```
Then once CLI is started, you can use command `help` to see all the available commands.
### Creating many users
For create many users you can use command `createUsers`
For example this will create 500 users `test0, test1, test2, ... , test499` in realm `demo` and each 100 users in separate transaction. All users will be granted realm roles `user` and `admin` :
```
createUsers test test demo 0 500 100 user,admin
```
Check count of users:
```
getUsersCount demo
```
Check if concrete user was really created:
```
getUser demo test499
```
### Creating many offline sessions
For create many offline sessions you can use command `persistSessions` . For example create 50000 sessions (each 500 in separate transaction) with command:
```
persistSessions 50000 500
```
Once users or sessions are created, you can restart to ensure the startup import of offline sessions will be triggered and you can see impact on startup time. After restart you can use command:
```
size
```
to doublecheck total count of sessions in infinispan (it will be 2 times as there is also 1 client session per each user session created)

4
model/jpa/src/main/resources/META-INF/jpa-changelog-1.9.0.xml
View file

@ -73,5 +73,9 @@
<dropUniqueConstraint tableName="FED_PROVIDERS" constraintName="UK_DCCIRJLIPU1478VQC89DID88C" />
<dropTable tableName="FED_PROVIDERS" />
<createIndex indexName="IDX_US_SESS_ID_ON_CL_SESS" tableName="OFFLINE_CLIENT_SESSION">
<column name="USER_SESSION_ID" type="VARCHAR(36)"/>
</createIndex>
</changeSet>
</databaseChangeLog>

10
testsuite/integration/pom.xml
View file

@ -237,16 +237,6 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>${postgresql.version}</version>
</dependency>
</dependencies>
<build>
<plugins>

6
testsuite/integration/src/test/java/org/keycloak/testsuite/KeycloakServer.java
View file

@ -33,7 +33,7 @@ import org.keycloak.services.filters.KeycloakSessionServletFilter;
import org.keycloak.services.managers.ApplianceBootstrap;
import org.keycloak.services.managers.RealmManager;
import org.keycloak.services.resources.KeycloakApplication;
import org.keycloak.testsuite.util.cli.InfinispanCLI;
import org.keycloak.testsuite.util.cli.TestsuiteCLI;
import org.keycloak.util.JsonSerialization;
import javax.servlet.DispatcherType;
@ -206,8 +206,8 @@ public class KeycloakServer {
}
});
if (System.getProperties().containsKey("startInfinispanCLI")) {
new InfinispanCLI(keycloak).start();
if (System.getProperties().containsKey("startTestsuiteCLI")) {
new TestsuiteCLI(keycloak).start();
}
return keycloak;

2
testsuite/integration/src/test/java/org/keycloak/testsuite/util/cli/AbstractCommand.java
View file

@ -35,7 +35,7 @@ public abstract class AbstractCommand {
protected List<String> args;
protected KeycloakSessionFactory sessionFactory;
public void injectProperties(List<String> args, InfinispanCLI cli, KeycloakSessionFactory sessionFactory) {
public void injectProperties(List<String> args, TestsuiteCLI cli, KeycloakSessionFactory sessionFactory) {
this.args = args;
this.sessionFactory = sessionFactory;
}

53
testsuite/integration/src/test/java/org/keycloak/testsuite/util/cli/PersistSessionsCommand.java
View file

@ -43,10 +43,32 @@ public class PersistSessionsCommand extends AbstractCommand {
@Override
public void doRunCommand(KeycloakSession sess) {
final int count = getIntArg(0);
final int batchCount = getIntArg(1);
int remaining = count;
while (remaining > 0) {
int createInThisBatch = Math.min(batchCount, remaining);
createSessionsBatch(createInThisBatch);
remaining = remaining - createInThisBatch;
}
// Write some summary
KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
@Override
public void run(KeycloakSession session) {
UserSessionPersisterProvider persister = session.getProvider(UserSessionPersisterProvider.class);
log.info("Command finished. Total number of sessions in persister: " + persister.getUserSessionsCount(true));
}
});
}
private void createSessionsBatch(final int countInThisBatch) {
final List<String> userSessionIds = new LinkedList<>();
final List<String> clientSessionIds = new LinkedList<>();
// Create sessions in separate transaction first
KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
@Override
@ -56,7 +78,7 @@ public class PersistSessionsCommand extends AbstractCommand {
ClientModel testApp = realm.getClientByClientId("security-admin-console");
UserSessionPersisterProvider persister = session.getProvider(UserSessionPersisterProvider.class);
for (int i = 0; i < count; i++) {
for (int i = 0; i < countInThisBatch; i++) {
UserSessionModel userSession = session.sessions().createUserSession(realm, john, "john-doh@localhost", "127.0.0.2", "form", true, null, null);
ClientSessionModel clientSession = session.sessions().createClientSession(realm, testApp);
clientSession.setUserSession(userSession);
@ -69,9 +91,10 @@ public class PersistSessionsCommand extends AbstractCommand {
});
log.info("Sessions created in infinispan storage");
log.infof("%d sessions created in infinispan storage", countInThisBatch);
// Persist them now
KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
@Override
@ -84,35 +107,17 @@ public class PersistSessionsCommand extends AbstractCommand {
counter++;
UserSessionModel userSession = session.sessions().getUserSession(realm, userSessionId);
persister.createUserSession(userSession, true);
if (counter%1000 == 0) {
log.infof("%d user sessions persisted. Continue", counter);
}
}
log.infof("All %d user sessions persisted", counter);
log.infof("%d user sessions persisted. Continue", counter);
counter = 0;
for (String clientSessionId : clientSessionIds) {
counter++;
ClientSessionModel clientSession = session.sessions().getClientSession(realm, clientSessionId);
persister.createClientSession(clientSession, true);
if (counter%1000 == 0) {
log.infof("%d client sessions persisted. Continue", counter);
}
}
log.infof("All %d client sessions persisted", counter);
}
});
// Persist them now
KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() {
@Override
public void run(KeycloakSession session) {
UserSessionPersisterProvider persister = session.getProvider(UserSessionPersisterProvider.class);
log.info("Total number of sessions in persister: " + persister.getUserSessionsCount(true));
log.infof("%d client sessions persisted. Continue", counter);
}
});
@ -120,6 +125,6 @@ public class PersistSessionsCommand extends AbstractCommand {
@Override
public String printUsage() {
return super.printUsage() + " <sessions-count>";
return super.printUsage() + " <sessions-count> <sessions-count-per-each-transaction>";
}
}

28
testsuite/integration/src/test/java/org/keycloak/testsuite/util/cli/InfinispanCLI.java → testsuite/integration/src/test/java/org/keycloak/testsuite/util/cli/TestsuiteCLI.java
View file

@ -32,27 +32,13 @@ import org.keycloak.models.KeycloakSessionFactory;
import org.keycloak.testsuite.KeycloakServer;
/**
* HOWTO USE THIS:
*
* 1) Run KeycloakServer with system properties (assuming mongo up and running on localhost):
* -Dkeycloak.realm.provider=mongo -Dkeycloak.user.provider=mongo -Dkeycloak.userSessionPersister.provider=mongo -Dkeycloak.connectionsMongo.db=keycloak -Dkeycloak.connectionsInfinispan.clustered=true -Dresources -DstartInfinispanCLI
*
* 2) Write command on STDIN to persist 50000 userSessions to mongo: persistSessions 50000
*
* 3) Run command "clear" to ensure infinispan cache is cleared. Doublecheck with command "size" is 0
*
* 4) Write command to load sessions from persistent storage - 100 sessions per worker transaction: loadPersistentSessions 100
*
* See the progress in log. Finally run command "size" to ensure size is 100001 (50000 userSessions + 50000 clientSessions + 1 initializationState item)
*
* 5) Alternative to step 3+4 - Kill the server after step 2 and start two KeycloakServer in parallel on ports 8081 and 8082 . See the progress in logs of loading persistent sessions to infinispan.
* Kill the coordinator (usually 8081 node) during startup and see the node 8082 became coordinator and took ownership of loading persistent sessions. After node 8082 fully started, the size of infinispan is again 100001
* See Testsuite.md (section how to create many users and offline sessions)
*
* @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
*/
public class InfinispanCLI {
public class TestsuiteCLI {
private static final Logger log = Logger.getLogger(InfinispanCLI.class);
private static final Logger log = Logger.getLogger(TestsuiteCLI.class);
private static final Class<?>[] BUILTIN_COMMANDS = {
ExitCommand.class,
@ -76,7 +62,7 @@ public class InfinispanCLI {
private final KeycloakSessionFactory sessionFactory;
private final Map<String, Class<? extends AbstractCommand>> commands = new LinkedHashMap<>();
public InfinispanCLI(KeycloakServer server) {
public TestsuiteCLI(KeycloakServer server) {
this.sessionFactory = server.getSessionFactory();
// register builtin commands
@ -97,7 +83,7 @@ public class InfinispanCLI {
// WARNING: Stdin blocking operation
public void start() throws IOException {
log.info("Starting infinispan CLI. Exit with 'exit' . Available commands with 'help' ");
log.info("Starting testsuite CLI. Exit with 'exit' . Available commands with 'help' ");
BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));
String line;
@ -132,7 +118,7 @@ public class InfinispanCLI {
System.out.print("$ ");
}
} finally {
log.info("Exit infinispan CLI");
log.info("Exit testsuite CLI");
reader.close();
}
}
@ -165,7 +151,7 @@ public class InfinispanCLI {
private List<String> commandNames = new ArrayList<>();
@Override
public void injectProperties(List<String> args, InfinispanCLI cli, KeycloakSessionFactory sessionFactory) {
public void injectProperties(List<String> args, TestsuiteCLI cli, KeycloakSessionFactory sessionFactory) {
for (String commandName : cli.commands.keySet()) {
commandNames.add(commandName);
}