From 955cbc76d78b92c81255decd68cab06b4517d3a4 Mon Sep 17 00:00:00 2001 From: Hynek Mlnarik Date: Mon, 26 Jun 2017 09:55:21 +0200 Subject: [PATCH] KEYCLOAK-5030 Change action tokens cache type to distributed --- .../content/bin/migrate-domain-clustered.cli | 14 +++---- .../content/bin/migrate-standalone-ha.cli | 14 +++---- .../src/main/cli/keycloak-install-ha-base.cli | 6 +-- ...ltInfinispanConnectionProviderFactory.java | 37 +++++++++++++++++-- .../InfinispanActionTokenStoreProvider.java | 8 ++-- ...nispanActionTokenStoreProviderFactory.java | 23 +++++------- .../jboss/common/add-keycloak-caches.xsl | 2 + .../crossdc/AbstractAdminCrossDCTest.java | 1 - .../crossdc/AbstractCrossDCTest.java | 8 ++-- .../crossdc/ActionTokenCrossDCTest.java | 18 ++++----- .../base/src/test/resources/arquillian.xml | 2 +- .../keycloak-infinispan.xml | 4 +- .../keycloak-infinispan2.xml | 4 +- 13 files changed, 84 insertions(+), 57 deletions(-) diff --git a/distribution/feature-packs/server-feature-pack/src/main/resources/content/bin/migrate-domain-clustered.cli b/distribution/feature-packs/server-feature-pack/src/main/resources/content/bin/migrate-domain-clustered.cli index 0f477458de..934421794e 100644 --- a/distribution/feature-packs/server-feature-pack/src/main/resources/content/bin/migrate-domain-clustered.cli +++ b/distribution/feature-packs/server-feature-pack/src/main/resources/content/bin/migrate-domain-clustered.cli @@ -206,13 +206,13 @@ if (outcome == failed) of /profile=$clusteredProfile/subsystem=infinispan/cache- echo end-if -if (outcome == failed) of /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/:read-resource - echo Adding local-cache=actionTokens to keycloak cache container... - /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/:add(indexing=NONE,start=LAZY) - /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/component=eviction/:write-attribute(name=strategy,value=NONE) - /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/component=eviction/:write-attribute(name=max-entries,value=-1) - /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/component=expiration/:write-attribute(name=interval,value=300000) - /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/component=expiration/:write-attribute(name=max-idle,value=-1) +if (outcome == failed) of /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/:read-resource + echo Adding distributed-cache=actionTokens to keycloak cache container... + /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/:add(indexing=NONE,mode=SYNC,owners=2) + /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/component=eviction/:write-attribute(name=strategy,value=NONE) + /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/component=eviction/:write-attribute(name=max-entries,value=-1) + /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/component=expiration/:write-attribute(name=interval,value=300000) + /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/component=expiration/:write-attribute(name=max-idle,value=-1) echo end-if diff --git a/distribution/feature-packs/server-feature-pack/src/main/resources/content/bin/migrate-standalone-ha.cli b/distribution/feature-packs/server-feature-pack/src/main/resources/content/bin/migrate-standalone-ha.cli index 4d5fac67db..4f4e3e09fa 100644 --- a/distribution/feature-packs/server-feature-pack/src/main/resources/content/bin/migrate-standalone-ha.cli +++ b/distribution/feature-packs/server-feature-pack/src/main/resources/content/bin/migrate-standalone-ha.cli @@ -211,13 +211,13 @@ if (outcome == failed) of /subsystem=infinispan/cache-container=keycloak/distrib echo end-if -if (outcome == failed) of /subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/:read-resource - echo Adding local-cache=actionTokens to keycloak cache container... - /subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/:add(indexing=NONE,start=LAZY) - /subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/component=eviction/:write-attribute(name=strategy,value=NONE) - /subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/component=eviction/:write-attribute(name=max-entries,value=-1) - /subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/component=expiration/:write-attribute(name=interval,value=300000) - /subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/component=expiration/:write-attribute(name=max-idle,value=-1) +if (outcome == failed) of /subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/:read-resource + echo Adding distributed-cache=actionTokens to keycloak cache container... + /subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/:add(indexing=NONE,mode=SYNC,owners=2) + /subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/component=eviction/:write-attribute(name=strategy,value=NONE) + /subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/component=eviction/:write-attribute(name=max-entries,value=-1) + /subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/component=expiration/:write-attribute(name=interval,value=300000) + /subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/component=expiration/:write-attribute(name=max-idle,value=-1) echo end-if diff --git a/distribution/server-overlay/src/main/cli/keycloak-install-ha-base.cli b/distribution/server-overlay/src/main/cli/keycloak-install-ha-base.cli index 4710eb8a82..6bb11d57d4 100644 --- a/distribution/server-overlay/src/main/cli/keycloak-install-ha-base.cli +++ b/distribution/server-overlay/src/main/cli/keycloak-install-ha-base.cli @@ -16,7 +16,7 @@ embed-server --server-config=standalone-ha.xml /subsystem=infinispan/cache-container=keycloak/local-cache=keys:add() /subsystem=infinispan/cache-container=keycloak/local-cache=keys/eviction=EVICTION:add(max-entries=1000,strategy=LRU) /subsystem=infinispan/cache-container=keycloak/local-cache=keys/expiration=EXPIRATION:add(max-idle=3600000) -/subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens:add() -/subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/eviction=EVICTION:add(max-entries=-1,strategy=NONE) -/subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/expiration=EXPIRATION:add(max-idle=-1,interval=300000) +/subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens:add(indexing="NONE",mode="SYNC",owners="2") +/subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/eviction=EVICTION:add(max-entries=-1,strategy=NONE) +/subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/expiration=EXPIRATION:add(max-idle=-1,interval=300000) /extension=org.keycloak.keycloak-server-subsystem/:add(module=org.keycloak.keycloak-server-subsystem) diff --git a/model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java b/model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java index 53e496f06c..a9df0471c1 100755 --- a/model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java +++ b/model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java @@ -248,7 +248,14 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon cacheManager.defineConfiguration(InfinispanConnectionProvider.KEYS_CACHE_NAME, getKeysCacheConfig()); cacheManager.getCache(InfinispanConnectionProvider.KEYS_CACHE_NAME, true); - cacheManager.defineConfiguration(InfinispanConnectionProvider.ACTION_TOKEN_CACHE, getActionTokenCacheConfig()); + final ConfigurationBuilder actionTokenCacheConfigBuilder = getActionTokenCacheConfig(); + if (clustered) { + actionTokenCacheConfigBuilder.clustering().cacheMode(async ? CacheMode.REPL_ASYNC : CacheMode.REPL_SYNC); + } + if (jdgEnabled) { + configureRemoteActionTokenCacheStore(actionTokenCacheConfigBuilder, async); + } + cacheManager.defineConfiguration(InfinispanConnectionProvider.ACTION_TOKEN_CACHE, actionTokenCacheConfigBuilder.build()); cacheManager.getCache(InfinispanConnectionProvider.ACTION_TOKEN_CACHE, true); long authzRevisionsMaxEntries = cacheManager.getCache(InfinispanConnectionProvider.AUTHORIZATION_CACHE_NAME).getCacheConfiguration().eviction().maxEntries(); @@ -301,6 +308,30 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon } + private void configureRemoteActionTokenCacheStore(ConfigurationBuilder builder, boolean async) { + String jdgServer = config.get("remoteStoreServer", "localhost"); + Integer jdgPort = config.getInt("remoteStorePort", 11222); + + builder.persistence() + .passivation(false) + .addStore(RemoteStoreConfigurationBuilder.class) + .fetchPersistentState(false) + .ignoreModifications(false) + .purgeOnStartup(false) + .preload(true) + .shared(true) + .remoteCacheName(InfinispanConnectionProvider.ACTION_TOKEN_CACHE) + .rawValues(true) + .forceReturnValues(false) + .marshaller(KeycloakHotRodMarshallerFactory.class.getName()) + .addServer() + .host(jdgServer) + .port(jdgPort) + .async() + .enabled(async); + + } + protected Configuration getKeysCacheConfig() { ConfigurationBuilder cb = new ConfigurationBuilder(); cb.eviction().strategy(EvictionStrategy.LRU).type(EvictionType.COUNT).size(InfinispanConnectionProvider.KEYS_CACHE_DEFAULT_MAX); @@ -308,7 +339,7 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon return cb.build(); } - private Configuration getActionTokenCacheConfig() { + private ConfigurationBuilder getActionTokenCacheConfig() { ConfigurationBuilder cb = new ConfigurationBuilder(); cb.eviction() @@ -319,7 +350,7 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon .maxIdle(InfinispanConnectionProvider.ACTION_TOKEN_MAX_IDLE_SECONDS, TimeUnit.SECONDS) .wakeUpInterval(InfinispanConnectionProvider.ACTION_TOKEN_WAKE_UP_INTERVAL_SECONDS, TimeUnit.SECONDS); - return cb.build(); + return cb; } private static final Object CHANNEL_INIT_SYNCHRONIZER = new Object(); diff --git a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanActionTokenStoreProvider.java b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanActionTokenStoreProvider.java index f02fb5a403..b4689aa743 100644 --- a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanActionTokenStoreProvider.java +++ b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanActionTokenStoreProvider.java @@ -17,13 +17,14 @@ package org.keycloak.models.sessions.infinispan; import org.keycloak.cluster.ClusterProvider; +import org.keycloak.common.util.Time; import org.keycloak.models.*; -import org.keycloak.models.cache.infinispan.events.AddInvalidatedActionTokenEvent; import org.keycloak.models.cache.infinispan.events.RemoveActionTokensSpecificEvent; import org.keycloak.models.sessions.infinispan.entities.ActionTokenValueEntity; import org.keycloak.models.sessions.infinispan.entities.ActionTokenReducedKey; import java.util.*; +import java.util.concurrent.TimeUnit; import org.infinispan.Cache; /** @@ -57,9 +58,7 @@ public class InfinispanActionTokenStoreProvider implements ActionTokenStoreProvi ActionTokenReducedKey tokenKey = new ActionTokenReducedKey(key.getUserId(), key.getActionId(), key.getActionVerificationNonce()); ActionTokenValueEntity tokenValue = new ActionTokenValueEntity(notes); - ClusterProvider cluster = session.getProvider(ClusterProvider.class); - AddInvalidatedActionTokenEvent event = new AddInvalidatedActionTokenEvent(tokenKey, key.getExpiration(), tokenValue); - this.tx.notify(cluster, generateActionTokenEventId(), event, false); + this.tx.put(actionKeyCache, tokenKey, tokenValue, key.getExpiration() - Time.currentTime(), TimeUnit.SECONDS); } private static String generateActionTokenEventId() { @@ -92,6 +91,7 @@ public class InfinispanActionTokenStoreProvider implements ActionTokenStoreProvi return value; } + @Override public void removeAll(String userId, String actionId) { if (userId == null || actionId == null) { return; diff --git a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanActionTokenStoreProviderFactory.java b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanActionTokenStoreProviderFactory.java index f67f28f6c0..95ee903507 100644 --- a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanActionTokenStoreProviderFactory.java +++ b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanActionTokenStoreProviderFactory.java @@ -19,16 +19,16 @@ package org.keycloak.models.sessions.infinispan; import org.keycloak.Config; import org.keycloak.Config.Scope; import org.keycloak.cluster.ClusterProvider; -import org.keycloak.common.util.Time; import org.keycloak.connections.infinispan.InfinispanConnectionProvider; import org.keycloak.models.*; -import org.keycloak.models.cache.infinispan.events.AddInvalidatedActionTokenEvent; import org.keycloak.models.cache.infinispan.events.RemoveActionTokensSpecificEvent; import org.keycloak.models.sessions.infinispan.entities.ActionTokenValueEntity; import org.keycloak.models.sessions.infinispan.entities.ActionTokenReducedKey; +import java.util.List; import java.util.Objects; -import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.infinispan.AdvancedCache; import org.infinispan.Cache; import org.infinispan.context.Flag; import org.infinispan.remoting.transport.Address; @@ -76,22 +76,17 @@ public class InfinispanActionTokenStoreProviderFactory implements ActionTokenSto LOG.debugf("[%s] Removing token invalidation for user+action: userId=%s, actionId=%s", cacheAddress, e.getUserId(), e.getActionId()); - cache + AdvancedCache localCache = cache .getAdvancedCache() - .withFlags(Flag.CACHE_MODE_LOCAL, Flag.SKIP_CACHE_LOAD) + .withFlags(Flag.CACHE_MODE_LOCAL, Flag.SKIP_CACHE_LOAD); + + List toRemove = localCache .keySet() .stream() .filter(k -> Objects.equals(k.getUserId(), e.getUserId()) && Objects.equals(k.getActionId(), e.getActionId())) - .forEach(cache::remove); - } else if (event instanceof AddInvalidatedActionTokenEvent) { - AddInvalidatedActionTokenEvent e = (AddInvalidatedActionTokenEvent) event; + .collect(Collectors.toList()); - LOG.debugf("[%s] Invalidating token %s", cacheAddress, e.getKey()); - if (e.getExpirationInSecs() == DEFAULT_CACHE_EXPIRATION) { - cache.put(e.getKey(), e.getTokenValue()); - } else { - cache.put(e.getKey(), e.getTokenValue(), e.getExpirationInSecs() - Time.currentTime(), TimeUnit.SECONDS); - } + toRemove.forEach(localCache::remove); } }); diff --git a/testsuite/integration-arquillian/servers/cache-server/jboss/common/add-keycloak-caches.xsl b/testsuite/integration-arquillian/servers/cache-server/jboss/common/add-keycloak-caches.xsl index ee9c29d419..540b4b5f57 100644 --- a/testsuite/integration-arquillian/servers/cache-server/jboss/common/add-keycloak-caches.xsl +++ b/testsuite/integration-arquillian/servers/cache-server/jboss/common/add-keycloak-caches.xsl @@ -30,6 +30,7 @@ + @@ -38,6 +39,7 @@ + diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractAdminCrossDCTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractAdminCrossDCTest.java index 2baa336cf9..2ad3cc3517 100644 --- a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractAdminCrossDCTest.java +++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractAdminCrossDCTest.java @@ -28,7 +28,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.BiConsumer; -import java.util.function.Consumer; import java.util.function.Function; import org.hamcrest.Matcher; import org.junit.Before; diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractCrossDCTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractCrossDCTest.java index c88c0c14aa..b4d4236a6d 100644 --- a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractCrossDCTest.java +++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractCrossDCTest.java @@ -116,11 +116,11 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest * @return */ protected Keycloak getAdminClientFor(ContainerInfo node) { - Keycloak adminClient = backendAdminClients.get(node); - if (adminClient == null && node.equals(suiteContext.getAuthServerInfo())) { - adminClient = this.adminClient; + Keycloak client = backendAdminClients.get(node); + if (client == null && node.equals(suiteContext.getAuthServerInfo())) { + client = this.adminClient; } - return adminClient; + return client; } /** diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/ActionTokenCrossDCTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/ActionTokenCrossDCTest.java index dbef2fcc9b..972be313dc 100644 --- a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/ActionTokenCrossDCTest.java +++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/ActionTokenCrossDCTest.java @@ -34,7 +34,6 @@ import javax.mail.internet.MimeMessage; import javax.ws.rs.core.Response; import org.jboss.arquillian.graphene.page.Page; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -45,7 +44,6 @@ import org.keycloak.testsuite.arquillian.InfinispanStatistics.Constants; import java.util.concurrent.TimeUnit; import org.hamcrest.Matchers; import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertThat; /** @@ -117,21 +115,23 @@ public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest { old -> greaterThan((Comparable) 0l) ); - // Verify that the caches are synchronized - assertThat(cacheDc0Node0Statistics.getSingleStatistics(Constants.STAT_CACHE_NUMBER_OF_ENTRIES), greaterThan(originalNumberOfEntries)); - assertThat(cacheDc0Node0Statistics.getSingleStatistics(Constants.STAT_CACHE_NUMBER_OF_ENTRIES), - is(cacheDc1Node0Statistics.getSingleStatistics(Constants.STAT_CACHE_NUMBER_OF_ENTRIES))); - assertEquals("Your account has been updated.", driver.getTitle()); + // Verify that there was an action token added in the node which was targetted by the link + assertThat(cacheDc0Node0Statistics.getSingleStatistics(Constants.STAT_CACHE_NUMBER_OF_ENTRIES), greaterThan(originalNumberOfEntries)); + disableDcOnLoadBalancer(0); enableDcOnLoadBalancer(1); - driver.navigate().to(link); + // Make sure that after going to the link, the invalidated action token has been retrieved from Infinispan server cluster in the other DC + assertSingleStatistics(cacheDc1Node0Statistics, Constants.STAT_CACHE_NUMBER_OF_ENTRIES, + () -> driver.navigate().to(link), + Matchers::greaterThan + ); + errorPage.assertCurrent(); } - @Ignore("KEYCLOAK-5030") @Test public void sendResetPasswordEmailAfterNewNodeAdded() throws IOException, MessagingException { disableDcOnLoadBalancer(1); diff --git a/testsuite/integration-arquillian/tests/base/src/test/resources/arquillian.xml b/testsuite/integration-arquillian/tests/base/src/test/resources/arquillian.xml index f9919d5867..58ef272b7c 100644 --- a/testsuite/integration-arquillian/tests/base/src/test/resources/arquillian.xml +++ b/testsuite/integration-arquillian/tests/base/src/test/resources/arquillian.xml @@ -224,7 +224,7 @@ org.keycloak.testsuite.arquillian.undertow.lb.SimpleUndertowLoadBalancerContainer localhost ${auth.server.http.port} - auth-server-undertow-cross-dc-0.1=http://localhost:8101,auth-server-undertow-cross-dc-0.2-manual=http://localhost:8102,auth-server-undertow-cross-dc-1.1=http://localhost:8111,auth-server-undertow-cross-dc-1.2-manual=http://localhost:8112 + auth-server-undertow-cross-dc-0_1=http://localhost:8101,auth-server-undertow-cross-dc-0_2-manual=http://localhost:8102,auth-server-undertow-cross-dc-1_1=http://localhost:8111,auth-server-undertow-cross-dc-1_2-manual=http://localhost:8112 diff --git a/wildfly/server-subsystem/src/main/resources/subsystem-templates/keycloak-infinispan.xml b/wildfly/server-subsystem/src/main/resources/subsystem-templates/keycloak-infinispan.xml index 818626a8fb..476449ab9c 100755 --- a/wildfly/server-subsystem/src/main/resources/subsystem-templates/keycloak-infinispan.xml +++ b/wildfly/server-subsystem/src/main/resources/subsystem-templates/keycloak-infinispan.xml @@ -113,10 +113,10 @@ - + - + diff --git a/wildfly/server-subsystem/src/main/resources/subsystem-templates/keycloak-infinispan2.xml b/wildfly/server-subsystem/src/main/resources/subsystem-templates/keycloak-infinispan2.xml index 839ecdf2df..ca801028f3 100755 --- a/wildfly/server-subsystem/src/main/resources/subsystem-templates/keycloak-infinispan2.xml +++ b/wildfly/server-subsystem/src/main/resources/subsystem-templates/keycloak-infinispan2.xml @@ -116,10 +116,10 @@ - + - +