From 4193856fbb51f7b8888f06cf6d153626581beb7a Mon Sep 17 00:00:00 2001 From: mposolda Date: Mon, 21 Mar 2016 11:19:39 +0100 Subject: [PATCH 1/3] KEYCLOAK-2687 ZIPException during import big number of users --- .../src/main/java/org/keycloak/common/util/Base64.java | 9 +++++++-- .../org/keycloak/exportimport/dir/DirImportProvider.java | 3 ++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/common/src/main/java/org/keycloak/common/util/Base64.java b/common/src/main/java/org/keycloak/common/util/Base64.java index 163bf2eead..b90f1b29c0 100644 --- a/common/src/main/java/org/keycloak/common/util/Base64.java +++ b/common/src/main/java/org/keycloak/common/util/Base64.java @@ -561,7 +561,7 @@ public class Base64 while( raw.hasRemaining() ){ int rem = Math.min(3,raw.remaining()); raw.get(raw3,0,rem); - Base64.encode3to4(enc4, raw3, rem, Base64.NO_OPTIONS ); + Base64.encode3to4(enc4, raw3, rem, Base64.NO_OPTIONS); encoded.put(enc4); } // end input remaining } @@ -1282,7 +1282,12 @@ public class Base64 } // end try catch( java.io.IOException e ) { - e.printStackTrace(); + if (e.getMessage().equals("Unsupported compression method")) { + System.out.println("Base64 decoding: Ignoring GZIP header and just returning originally-decoded bytes."); // Better to log as debug, but jboss logging not available in the module :/ + } else { + e.printStackTrace(); + } + // Just return originally-decoded bytes } // end catch finally { diff --git a/services/src/main/java/org/keycloak/exportimport/dir/DirImportProvider.java b/services/src/main/java/org/keycloak/exportimport/dir/DirImportProvider.java index 329b7297c3..391d9a68c4 100755 --- a/services/src/main/java/org/keycloak/exportimport/dir/DirImportProvider.java +++ b/services/src/main/java/org/keycloak/exportimport/dir/DirImportProvider.java @@ -133,12 +133,13 @@ public class DirImportProvider implements ImportProvider { if (realmImported.get()) { // Import users - for (File userFile : userFiles) { + for (final File userFile : userFiles) { final FileInputStream fis = new FileInputStream(userFile); KeycloakModelUtils.runJobInTransaction(factory, new ExportImportSessionTask() { @Override protected void runExportImportTask(KeycloakSession session) throws IOException { ImportUtils.importUsersFromStream(session, realmName, JsonSerialization.mapper, fis); + logger.infof("Imported users from %s", userFile.getAbsolutePath()); } }); } From e2e96f7183a5c692a427212f94640377cfcb4328 Mon Sep 17 00:00:00 2001 From: mposolda Date: Mon, 21 Mar 2016 13:04:30 +0100 Subject: [PATCH 2/3] KEYCLOAK-2630 Add caching for user consents --- .../models/cache/infinispan/UserAdapter.java | 66 ++++++++++++++++--- .../cache/infinispan/entities/CachedUser.java | 16 ++++- .../entities/CachedUserConsent.java | 55 ++++++++++++++++ 3 files changed, 127 insertions(+), 10 deletions(-) create mode 100644 model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/entities/CachedUserConsent.java diff --git a/model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/UserAdapter.java b/model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/UserAdapter.java index 6e53626592..2b02cee883 100755 --- a/model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/UserAdapter.java +++ b/model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/UserAdapter.java @@ -17,12 +17,28 @@ package org.keycloak.models.cache.infinispan; -import org.keycloak.models.*; -import org.keycloak.models.cache.CacheUserProvider; +import org.keycloak.models.ClientModel; +import org.keycloak.models.GroupModel; +import org.keycloak.models.KeycloakSession; +import org.keycloak.models.ProtocolMapperModel; +import org.keycloak.models.RealmModel; +import org.keycloak.models.RoleContainerModel; +import org.keycloak.models.RoleModel; +import org.keycloak.models.UserConsentModel; +import org.keycloak.models.UserCredentialModel; +import org.keycloak.models.UserCredentialValueModel; +import org.keycloak.models.UserModel; import org.keycloak.models.cache.infinispan.entities.CachedUser; +import org.keycloak.models.cache.infinispan.entities.CachedUserConsent; import org.keycloak.models.utils.KeycloakModelUtils; -import java.util.*; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; /** * @author Bill Burke @@ -380,16 +396,28 @@ public class UserAdapter implements UserModel { @Override public UserConsentModel getConsentByClient(String clientId) { - // TODO: caching? - getDelegateForUpdate(); - return updated.getConsentByClient(clientId); + if (updated != null) return updated.getConsentByClient(clientId); + CachedUserConsent cachedConsent = cached.getConsents().get(clientId); + if (cachedConsent == null) { + return null; + } + + return toConsentModel(cachedConsent); } @Override public List getConsents() { - // TODO: caching? - getDelegateForUpdate(); - return updated.getConsents(); + if (updated != null) return updated.getConsents(); + Collection cachedConsents = cached.getConsents().values(); + + List result = new LinkedList<>(); + for (CachedUserConsent cachedConsent : cachedConsents) { + UserConsentModel consent = toConsentModel(cachedConsent); + if (consent != null) { + result.add(consent); + } + } + return result; } @Override @@ -404,6 +432,26 @@ public class UserAdapter implements UserModel { return updated.revokeConsentForClient(clientId); } + private UserConsentModel toConsentModel(CachedUserConsent cachedConsent) { + ClientModel client = keycloakSession.realms().getClientById(cachedConsent.getClientDbId(), realm); + if (client == null) { + return null; + } + + UserConsentModel consentModel = new UserConsentModel(client); + + for (String roleId : cachedConsent.getRoleIds()) { + RoleModel role = keycloakSession.realms().getRoleById(roleId, realm); + if (role != null) { + consentModel.addGrantedRole(role); + } + } + for (ProtocolMapperModel protocolMapper : cachedConsent.getProtocolMappers()) { + consentModel.addGrantedProtocolMapper(protocolMapper); + } + return consentModel; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/entities/CachedUser.java b/model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/entities/CachedUser.java index af300001a6..f37ecf1936 100755 --- a/model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/entities/CachedUser.java +++ b/model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/entities/CachedUser.java @@ -20,14 +20,16 @@ package org.keycloak.models.cache.infinispan.entities; import org.keycloak.models.GroupModel; import org.keycloak.models.RealmModel; import org.keycloak.models.RoleModel; +import org.keycloak.models.UserConsentModel; import org.keycloak.models.UserCredentialValueModel; import org.keycloak.models.UserModel; import org.keycloak.common.util.MultivaluedHashMap; -import java.io.Serializable; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Set; /** @@ -51,6 +53,7 @@ public class CachedUser extends AbstractRevisioned implements InRealm { private Set requiredActions = new HashSet<>(); private Set roleMappings = new HashSet<>(); private Set groups = new HashSet<>(); + private Map consents = new HashMap<>(); // Key is client DB Id @@ -79,6 +82,13 @@ public class CachedUser extends AbstractRevisioned implements InRealm { groups.add(group.getId()); } } + + List consents = user.getConsents(); + if (consents != null) { + for (UserConsentModel consent : consents) { + this.consents.put(consent.getClient().getId(), new CachedUserConsent(consent)); + } + } } public String getRealm() { @@ -144,4 +154,8 @@ public class CachedUser extends AbstractRevisioned implements InRealm { public Set getGroups() { return groups; } + + public Map getConsents() { + return consents; + } } diff --git a/model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/entities/CachedUserConsent.java b/model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/entities/CachedUserConsent.java new file mode 100644 index 0000000000..00022acfc0 --- /dev/null +++ b/model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/entities/CachedUserConsent.java @@ -0,0 +1,55 @@ +/* + * Copyright 2016 Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.keycloak.models.cache.infinispan.entities; + +import java.util.HashSet; +import java.util.Set; + +import org.keycloak.models.ProtocolMapperModel; +import org.keycloak.models.RoleModel; +import org.keycloak.models.UserConsentModel; + +/** + * @author Marek Posolda + */ +public class CachedUserConsent { + + private final String clientDbId; + private final Set protocolMappers = new HashSet<>(); + private final Set roleIds = new HashSet<>(); + + public CachedUserConsent(UserConsentModel consentModel) { + this.clientDbId = consentModel.getClient().getId(); + this.protocolMappers.addAll(consentModel.getGrantedProtocolMappers()); + for (RoleModel role : consentModel.getGrantedRoles()) { + this.roleIds.add(role.getId()); + } + } + + public String getClientDbId() { + return clientDbId; + } + + public Set getProtocolMappers() { + return protocolMappers; + } + + public Set getRoleIds() { + return roleIds; + } +} From 5c912d3501ace75fb4acc8d72b347e0288f3ae77 Mon Sep 17 00:00:00 2001 From: mposolda Date: Tue, 22 Mar 2016 06:25:21 +0100 Subject: [PATCH 3/3] KEYCLOAK-2631 Caching of service account links --- .../cache/infinispan/UserCacheSession.java | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/UserCacheSession.java b/model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/UserCacheSession.java index c611bd37e4..17bc283317 100755 --- a/model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/UserCacheSession.java +++ b/model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/UserCacheSession.java @@ -18,7 +18,20 @@ package org.keycloak.models.cache.infinispan; import org.jboss.logging.Logger; -import org.keycloak.models.*; +import org.keycloak.common.constants.ServiceAccountConstants; +import org.keycloak.models.ClientModel; +import org.keycloak.models.CredentialValidationOutput; +import org.keycloak.models.FederatedIdentityModel; +import org.keycloak.models.GroupModel; +import org.keycloak.models.KeycloakSession; +import org.keycloak.models.KeycloakTransaction; +import org.keycloak.models.ProtocolMapperModel; +import org.keycloak.models.RealmModel; +import org.keycloak.models.RoleModel; +import org.keycloak.models.UserCredentialModel; +import org.keycloak.models.UserFederationProviderModel; +import org.keycloak.models.UserModel; +import org.keycloak.models.UserProvider; import org.keycloak.models.cache.CacheUserProvider; import org.keycloak.models.cache.infinispan.entities.CachedUser; import org.keycloak.models.cache.infinispan.entities.UserListQuery; @@ -281,6 +294,13 @@ public class UserCacheSession implements CacheUserProvider { @Override public UserModel getUserByServiceAccountClient(ClientModel client) { + // Just an attempt to find the user from cache by default serviceAccount username + String username = ServiceAccountConstants.SERVICE_ACCOUNT_USER_PREFIX + client.getClientId(); + UserModel user = getUserByUsername(username, client.getRealm()); + if (user != null && user.getServiceAccountClientLink() != null && user.getServiceAccountClientLink().equals(client.getId())) { + return user; + } + return getDelegate().getUserByServiceAccountClient(client); }