8947 - Add liquibase extension to handle JSON operations

This commit is contained in:
Stefan Guilhen 2021-11-18 17:24:36 -03:00 committed by Hynek Mlnařík
parent 208e45cfb2
commit b12830ae4f
19 changed files with 1177 additions and 35 deletions

View file

@ -0,0 +1,105 @@
/*
* Copyright 2021 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.models.map.storage.jpa.liquibase;
import java.sql.Connection;
import java.util.concurrent.atomic.AtomicBoolean;
import liquibase.Liquibase;
import liquibase.change.ChangeFactory;
import liquibase.database.Database;
import liquibase.database.DatabaseFactory;
import liquibase.database.jvm.JdbcConnection;
import liquibase.datatype.DataTypeFactory;
import liquibase.exception.LiquibaseException;
import liquibase.resource.ClassLoaderResourceAccessor;
import liquibase.resource.ResourceAccessor;
import liquibase.sqlgenerator.SqlGeneratorFactory;
import org.jboss.logging.Logger;
import org.keycloak.models.KeycloakSession;
import org.keycloak.models.map.storage.jpa.liquibase.extension.GeneratedColumnSqlGenerator;
import org.keycloak.models.map.storage.jpa.liquibase.extension.CreateJsonIndexChange;
import org.keycloak.models.map.storage.jpa.liquibase.extension.CreateJsonIndexGenerator;
import org.keycloak.models.map.storage.jpa.liquibase.extension.GeneratedColumnChange;
import org.keycloak.models.map.storage.jpa.liquibase.extension.JsonDataType;
/**
* A {@link MapLiquibaseConnectionProvider} implementation for the map-jpa module. This provider registers the custom {@code Liquibase}
* changes and data types that were developed to better support working with data stored as JSON in the database.
* </p>
* An instance of this provider can be obtained via {@link KeycloakSession#getProvider(Class)} as follows:
* <pre>
* MapLiquibaseConnectionProvider liquibaseProvider = session.getProvider(MapLiquibaseConnectionProvider.class);
* </pre>
*
* @author <a href="mailto:sguilhen@redhat.com">Stefan Guilhen</a>
*/
public class DefaultLiquibaseConnectionProvider implements MapLiquibaseConnectionProvider {
private static final Logger logger = Logger.getLogger(DefaultLiquibaseConnectionProvider.class);
private static final AtomicBoolean INITIALIZED = new AtomicBoolean(false);
public DefaultLiquibaseConnectionProvider(final KeycloakSession session) {
if (! INITIALIZED.get()) {
// TODO: all liquibase providers should probably synchronize on the same object.
synchronized (INITIALIZED) {
if (! INITIALIZED.get()) {
initializeLiquibase();
INITIALIZED.set(true);
}
}
}
}
/**
* Registers the custom changes/types so we can work with data stored in JSON format.
*/
protected void initializeLiquibase() {
// Add custom JSON data type
DataTypeFactory.getInstance().register(JsonDataType.class);
// Add custom change to generate columns from properties in JSON files stored in the DB.
ChangeFactory.getInstance().register(GeneratedColumnChange.class);
SqlGeneratorFactory.getInstance().register(new GeneratedColumnSqlGenerator());
// Add custom change to create indexes for properties in JSON files stored in the DB.
ChangeFactory.getInstance().register(CreateJsonIndexChange.class);
SqlGeneratorFactory.getInstance().register(new CreateJsonIndexGenerator());
}
@Override
public void close() {
}
@Override
public Liquibase getLiquibaseForCustomUpdate(final Connection connection, final String defaultSchema, final String changelogLocation,
final ClassLoader classloader, final String changelogTableName) throws LiquibaseException {
Database database = DatabaseFactory.getInstance().findCorrectDatabaseImplementation(new JdbcConnection(connection));
if (defaultSchema != null) {
database.setDefaultSchemaName(defaultSchema);
}
ResourceAccessor resourceAccessor = new ClassLoaderResourceAccessor(classloader);
database.setDatabaseChangeLogTableName(changelogTableName);
logger.debugf("Using changelog file %s and changelogTableName %s", changelogLocation, database.getDatabaseChangeLogTableName());
return new Liquibase(changelogLocation, resourceAccessor, database);
}
}

View file

@ -0,0 +1,55 @@
/*
* Copyright 2021 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.models.map.storage.jpa.liquibase;
import org.keycloak.Config;
import org.keycloak.models.KeycloakSession;
import org.keycloak.models.KeycloakSessionFactory;
/**
* {@link MapLiquibaseConnectionProviderFactory} implementation for the map-jpa module. It produces an instance of
* {@link DefaultLiquibaseConnectionProvider} when {@link #create(KeycloakSession)} is called.
*
* @author <a href="mailto:sguilhen@redhat.com">Stefan Guilhen</a>
*/
public class DefaultLiquibaseConnectionProviderFactory implements MapLiquibaseConnectionProviderFactory {
public static final String PROVIDER_ID = "default";
@Override
public MapLiquibaseConnectionProvider create(KeycloakSession session) {
return new DefaultLiquibaseConnectionProvider(session);
}
@Override
public void init(Config.Scope config) {
}
@Override
public void postInit(KeycloakSessionFactory factory) {
}
@Override
public void close() {
}
@Override
public String getId() {
return PROVIDER_ID;
}
}

View file

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.keycloak.models.map.storage.jpa.updater.liquibase;
package org.keycloak.models.map.storage.jpa.liquibase;
import java.io.File;
import java.io.FileWriter;
@ -33,12 +33,10 @@ import liquibase.changelog.RanChangeSet;
import liquibase.exception.LiquibaseException;
import org.jboss.logging.Logger;
import org.keycloak.common.util.reflections.Reflections;
import org.keycloak.connections.jpa.updater.liquibase.conn.LiquibaseConnectionProvider;
import org.keycloak.connections.jpa.updater.liquibase.ThreadLocalSessionContext;
import org.keycloak.models.KeycloakSession;
import org.keycloak.models.map.storage.ModelEntityUtil;
import org.keycloak.models.map.storage.jpa.updater.MapJpaUpdaterProvider;
import org.keycloak.models.map.storage.jpa.updater.MapJpaUpdaterProvider.Status;
public class MapJpaLiquibaseUpdaterProvider implements MapJpaUpdaterProvider {
@ -163,12 +161,12 @@ public class MapJpaLiquibaseUpdaterProvider implements MapJpaUpdaterProvider {
}
private Liquibase getLiquibase(Class modelType, Connection connection, String defaultSchema) throws LiquibaseException {
LiquibaseConnectionProvider liquibaseProvider = session.getProvider(LiquibaseConnectionProvider.class);
String changelog = "META-INF/jpa-" + ModelEntityUtil.getModelName(modelType) + "-changelog.xml";
if (changelog == null) {
MapLiquibaseConnectionProvider liquibaseProvider = session.getProvider(MapLiquibaseConnectionProvider.class);
String modelName = ModelEntityUtil.getModelName(modelType);
if (modelName == null) {
throw new IllegalStateException("Cannot find changlelog for modelClass " + modelType.getName());
}
String changelog = "META-INF/jpa-" + modelName + "-changelog.xml";
return liquibaseProvider.getLiquibaseForCustomUpdate(connection, defaultSchema, changelog, this.getClass().getClassLoader(), "databasechangelog");
}

View file

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.keycloak.models.map.storage.jpa.updater.liquibase;
package org.keycloak.models.map.storage.jpa.liquibase;
import org.keycloak.Config;
import org.keycloak.models.KeycloakSession;

View file

@ -0,0 +1,29 @@
/*
* Copyright 2021 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.models.map.storage.jpa.liquibase;
import java.sql.Connection;
import liquibase.Liquibase;
import liquibase.exception.LiquibaseException;
import org.keycloak.provider.Provider;
public interface MapLiquibaseConnectionProvider extends Provider {
Liquibase getLiquibaseForCustomUpdate(Connection connection, String defaultSchema, String changelogLocation, ClassLoader classloader, String changelogTableName) throws LiquibaseException;
}

View file

@ -0,0 +1,23 @@
/*
* Copyright 2021 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.models.map.storage.jpa.liquibase;
import org.keycloak.provider.ProviderFactory;
public interface MapLiquibaseConnectionProviderFactory extends ProviderFactory<MapLiquibaseConnectionProvider> {
}

View file

@ -0,0 +1,47 @@
/*
* Copyright 2021 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.models.map.storage.jpa.liquibase;
import org.keycloak.provider.Provider;
import org.keycloak.provider.ProviderFactory;
import org.keycloak.provider.Spi;
public class MapLiquibaseConnectionSpi implements Spi {
public final static String SPI_NAME = "mapLiquibaseConnection";
@Override
public boolean isInternal() {
return true;
}
@Override
public String getName() {
return SPI_NAME;
}
@Override
public Class<? extends Provider> getProviderClass() {
return MapLiquibaseConnectionProvider.class;
}
@Override
public Class<? extends ProviderFactory> getProviderFactoryClass() {
return MapLiquibaseConnectionProviderFactory.class;
}
}

View file

@ -0,0 +1,194 @@
/*
* Copyright 2021 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.models.map.storage.jpa.liquibase.extension;
import java.util.List;
import java.util.stream.Collectors;
import liquibase.change.AbstractChange;
import liquibase.change.Change;
import liquibase.change.ChangeMetaData;
import liquibase.change.ChangeStatus;
import liquibase.change.ChangeWithColumns;
import liquibase.change.DatabaseChange;
import liquibase.change.DatabaseChangeProperty;
import liquibase.change.core.CreateIndexChange;
import liquibase.change.core.DropIndexChange;
import liquibase.database.Database;
import liquibase.statement.SqlStatement;
/**
* Extension used to create an index for properties of JSON files stored in the database. Some databases, like {@code Postgres},
* have native support for these indexes while other databases may require different constructs to achieve this (like creation
* of a separate column based on the JSON property and subsequent indexing of that column).
* <p/>
* Example configuration in the changelog:
* <pre>
* &lt;databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
* xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
* xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
* http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.1.xsd
* http://www.liquibase.org/xml/ns/dbchangelog-ext
* http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd"&gt;
*
* &lt;changeSet author="keycloak" id="some_id"&gt;
* ...
* &lt;ext:createJsonIndex tableName="test" indexName="some_index_name"&gt;
* &lt;ext:column jsonColumn="metadata" jsonProperty="name"/&gt;
* &lt;/ext:createJsonIndex&gt;
* &lt;/changeSet&gt;
* </pre>
* The above configuration is creating an index for the {@code name} property of JSON files stored in column {@code metadata} in
* table {@code test}.
*
* @author <a href="mailto:sguilhen@redhat.com">Stefan Guilhen</a>
*/
@DatabaseChange(name="createJsonIndex", description = "Creates an index for one or more JSON properties",
priority = ChangeMetaData.PRIORITY_DEFAULT, appliesTo = "index")
public class CreateJsonIndexChange extends AbstractChange implements ChangeWithColumns<JsonEnabledColumnConfig> {
private final CreateIndexChange delegate;
public CreateJsonIndexChange() {
this.delegate = new CreateIndexChange();
}
@DatabaseChangeProperty
public String getCatalogName() {
return delegate.getCatalogName();
}
public void setCatalogName(String catalogName) {
this.delegate.setCatalogName(catalogName);
}
@DatabaseChangeProperty(mustEqualExisting ="index.schema")
public String getSchemaName() {
return delegate.getSchemaName();
}
public void setSchemaName(String schemaName) {
this.delegate.setSchemaName(schemaName);
}
@DatabaseChangeProperty(mustEqualExisting = "index.table", description = "Name of the table to add the index to", exampleValue = "person")
public String getTableName() {
return this.delegate.getTableName();
}
public void setTableName(String tableName) {
this.delegate.setTableName(tableName);
}
@DatabaseChangeProperty(mustEqualExisting = "index", description = "Name of the index to create")
public String getIndexName() {
return this.delegate.getIndexName();
}
public void setIndexName(String indexName) {
this.delegate.setIndexName(indexName);
}
@DatabaseChangeProperty(description = "Tablepace to create the index in.")
public String getTablespace() {
return this.delegate.getTablespace();
}
public void setTablespace(String tablespace) {
this.delegate.setTablespace(tablespace);
}
@DatabaseChangeProperty(description = "Unique values index", since = "1.8")
public Boolean isUnique() {
return this.delegate.isUnique();
}
public void setUnique(Boolean isUnique) {
this.delegate.setUnique(isUnique);
}
@DatabaseChangeProperty(isChangeProperty = false)
public String getAssociatedWith() {
return delegate.getAssociatedWith();
}
public void setAssociatedWith(String associatedWith) {
this.delegate.setAssociatedWith(associatedWith);
}
@DatabaseChangeProperty
public Boolean getClustered() {
return this.delegate.getClustered();
}
public void setClustered(Boolean clustered) {
this.delegate.setClustered(clustered);
}
@Override
public void addColumn(JsonEnabledColumnConfig column) {
delegate.addColumn(column);
}
@Override
@DatabaseChangeProperty(mustEqualExisting = "index.column", description = "Column(s) to add to the index", requiredForDatabase = "all")
public List<JsonEnabledColumnConfig> getColumns() {
return this.delegate.getColumns().stream().map(JsonEnabledColumnConfig.class::cast).collect(Collectors.toList());
}
@Override
public void setColumns(List<JsonEnabledColumnConfig> columns) {
columns.forEach(this.delegate::addColumn);
}
@Override
public String getConfirmationMessage() {
return delegate.getConfirmationMessage();
}
@Override
public SqlStatement[] generateStatements(Database database) {
return new SqlStatement[]{new CreateJsonIndexStatement(this.getCatalogName(), this.getSchemaName(), this.getTableName(),
this.getIndexName(), this.isUnique(), this.getAssociatedWith(), this.getTablespace(), this.getClustered(),
this.getColumns().toArray(new JsonEnabledColumnConfig[0]))};
}
@Override
protected Change[] createInverses() {
DropIndexChange inverse = new DropIndexChange();
inverse.setSchemaName(getSchemaName());
inverse.setTableName(getTableName());
inverse.setIndexName(getIndexName());
return new Change[]{inverse};
}
@Override
public ChangeStatus checkStatus(Database database) {
return delegate.checkStatus(database);
}
@Override
public String getSerializedObjectNamespace() {
return GENERIC_CHANGELOG_EXTENSION_NAMESPACE;
}
@Override
public Object getSerializableFieldValue(String field) {
return delegate.getSerializableFieldValue(field);
}
}

View file

@ -0,0 +1,112 @@
/*
* Copyright 2021 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.models.map.storage.jpa.liquibase.extension;
import java.util.Arrays;
import java.util.stream.Collectors;
import liquibase.database.Database;
import liquibase.database.core.PostgresDatabase;
import liquibase.exception.ValidationErrors;
import liquibase.sql.Sql;
import liquibase.sql.UnparsedSql;
import liquibase.sqlgenerator.SqlGenerator;
import liquibase.sqlgenerator.SqlGeneratorChain;
import liquibase.sqlgenerator.core.AbstractSqlGenerator;
import liquibase.statement.core.CreateIndexStatement;
import liquibase.structure.core.Index;
import liquibase.structure.core.Table;
import liquibase.util.StringUtils;
/**
* A {@link SqlGenerator} implementation that supports {@link CreateJsonIndexStatement}s. It generates the SQL required
* to create an index for properties of JSON files stored in one of the table columns.
*
* @author <a href="mailto:sguilhen@redhat.com">Stefan Guilhen</a>
*/
public class CreateJsonIndexGenerator extends AbstractSqlGenerator<CreateJsonIndexStatement> {
/**
* Override the priority. This is needed because {@link CreateJsonIndexStatement} is a subtype of {@link CreateIndexStatement}
* and is thus a match for the standard index generators. By increasing the priority we ensure this is processed before
* the other generators.
*
* @return this generator's priority.
*/
@Override
public int getPriority() {
return SqlGenerator.PRIORITY_DATABASE + 1;
}
@Override
public ValidationErrors validate(CreateJsonIndexStatement createIndexStatement, Database database, SqlGeneratorChain sqlGeneratorChain) {
ValidationErrors validationErrors = new ValidationErrors();
validationErrors.checkRequiredField("tableName", createIndexStatement.getTableName());
validationErrors.checkRequiredField("columns", createIndexStatement.getColumns());
Arrays.stream(createIndexStatement.getColumns()).map(JsonEnabledColumnConfig.class::cast)
.forEach(config -> {
validationErrors.checkRequiredField("jsonColumn", config.getJsonColumn());
validationErrors.checkRequiredField("jsonProperty", config.getJsonProperty());
});
return validationErrors;
}
@Override
public Sql[] generateSql(CreateJsonIndexStatement statement, Database database, SqlGeneratorChain sqlGeneratorChain) {
if (!(database instanceof PostgresDatabase)) {
// for now return an empty SQL for DBs that don't support JSON indexes natively.
return new Sql[0];
}
StringBuilder builder = new StringBuilder();
builder.append("CREATE ");
if (statement.isUnique() != null && statement.isUnique()) {
builder.append("UNIQUE ");
}
builder.append("INDEX ");
if (statement.getIndexName() != null) {
builder.append(database.escapeObjectName(statement.getIndexName(), Index.class)).append(" ");
}
builder.append("ON ").append(database.escapeTableName(statement.getTableCatalogName(), statement.getTableSchemaName(),
statement.getTableName()));
this.handleJsonIndex(statement, database, builder);
if (StringUtils.trimToNull(statement.getTablespace()) != null && database.supportsTablespaces()) {
builder.append(" TABLESPACE ").append(statement.getTablespace());
}
return new Sql[]{new UnparsedSql(builder.toString(), getAffectedIndex(statement))};
}
protected void handleJsonIndex(final CreateJsonIndexStatement statement, final Database database, final StringBuilder builder) {
if (database instanceof PostgresDatabase) {
builder.append(" USING gin (");
builder.append(Arrays.stream(statement.getColumns()).map(JsonEnabledColumnConfig.class::cast)
.map(c -> "(" + c.getJsonColumn() + "->'" + c.getJsonProperty() + "') jsonb_path_ops")
.collect(Collectors.joining(", ")))
.append(")");
}
}
protected Index getAffectedIndex(CreateIndexStatement statement) {
return new Index().setName(statement.getIndexName()).setTable((Table) new Table().setName(statement.getTableName())
.setSchema(statement.getTableCatalogName(), statement.getTableSchemaName()));
}
}

View file

@ -0,0 +1,39 @@
/*
* Copyright 2021 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.models.map.storage.jpa.liquibase.extension;
import liquibase.change.AddColumnConfig;
import liquibase.statement.core.CreateIndexStatement;
/**
* A {@link liquibase.statement.SqlStatement} that holds the information needed to create JSON indexes. Having a specific
* subtype allows for easier selection of the respective {@link liquibase.sqlgenerator.SqlGenerator}, since Liquibase
* selects the generators based on the statement type they are capable of handling.
*
* @author <a href="mailto:sguilhen@redhat.com">Stefan Guilhen</a>
*/
public class CreateJsonIndexStatement extends CreateIndexStatement {
public CreateJsonIndexStatement(final String tableCatalogName, final String tableSchemaName, final String tableName,
final String indexName, final Boolean isUnique, final String associatedWith,
final String tablespace, final Boolean clustered, final AddColumnConfig... columns) {
super(indexName, tableCatalogName, tableSchemaName, tableName, isUnique, associatedWith, columns);
super.setTablespace(tablespace);
super.setClustered(clustered);
}
}

View file

@ -0,0 +1,216 @@
/*
* Copyright 2021 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.models.map.storage.jpa.liquibase.extension;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
import liquibase.change.AbstractChange;
import liquibase.change.AddColumnConfig;
import liquibase.change.Change;
import liquibase.change.ChangeMetaData;
import liquibase.change.ChangeStatus;
import liquibase.change.ChangeWithColumns;
import liquibase.change.ColumnConfig;
import liquibase.change.DatabaseChange;
import liquibase.change.DatabaseChangeProperty;
import liquibase.change.core.AddColumnChange;
import liquibase.database.Database;
import liquibase.database.core.PostgresDatabase;
import liquibase.exception.ValidationErrors;
import liquibase.statement.SqlStatement;
import liquibase.statement.core.AddColumnStatement;
/**
* Extension used to a column whose values are generated from a property of a JSON file stored in one of the table's columns.
* <p/>
* Example configuration in the changelog:
* <pre>
* &lt;databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
* xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
* xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
* http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.1.xsd
* http://www.liquibase.org/xml/ns/dbchangelog-ext
* http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd"&gt;
*
* &lt;changeSet author="keycloak" id="some_id"&gt;
* ...
* &lt;ext:addGeneratedColumn tableName="test"&gt;
* &lt;ext:column name="new_column" type="VARCHAR(36)" jsonColumn="metadata" jsonProperty="alias"/&gt;
* &lt;/ext:addGeneratedColumn&gt;
* &lt;/changeSet&gt;
* </pre>
* The above configuration is adding a new column, named {@code new_column}, whose values are generated from the {@code alias} property
* of the JSON file stored in column {@code metadata}. If, for example, a particular entry in the table contains the JSON
* {@code {"name":"duke","alias":"jduke"}} in column {@code metadata}, the value generated for the new column will be {@code jduke}.
*
* @author <a href="mailto:sguilhen@redhat.com">Stefan Guilhen</a>
*/
@DatabaseChange(name = "addGeneratedColumn", description = "Adds new generated columns to a table. The columns must reference a JSON property inside an existing JSON column.",
priority = ChangeMetaData.PRIORITY_DEFAULT, appliesTo = "table")
public class GeneratedColumnChange extends AbstractChange implements ChangeWithColumns<JsonEnabledColumnConfig> {
private final ExtendedAddColumnChange delegate;
private Map<String, JsonEnabledColumnConfig> configMap = new HashMap<>();
public GeneratedColumnChange() {
this.delegate = new ExtendedAddColumnChange();
}
@DatabaseChangeProperty(mustEqualExisting ="relation.catalog")
public String getCatalogName() {
return this.delegate.getCatalogName();
}
public void setCatalogName(final String catalogName) {
this.delegate.setCatalogName(catalogName);
}
@DatabaseChangeProperty(mustEqualExisting ="relation.schema")
public String getSchemaName() {
return this.delegate.getSchemaName();
}
public void setSchemaName(final String schemaName) {
this.delegate.setSchemaName(schemaName);
}
@DatabaseChangeProperty(mustEqualExisting ="table", description = "Name of the table to add the generated column to")
public String getTableName() {
return this.delegate.getTableName();
}
public void setTableName(final String tableName) {
this.delegate.setTableName(tableName);
}
@Override
public void addColumn(final JsonEnabledColumnConfig column) {
this.delegate.addColumn(column);
this.configMap.put(column.getName(), column);
}
@Override
@DatabaseChangeProperty(description = "Generated columns information", requiredForDatabase = "all")
public List<JsonEnabledColumnConfig> getColumns() {
return this.delegate.getColumns().stream().map(JsonEnabledColumnConfig.class::cast).collect(Collectors.toList());
}
@Override
public void setColumns(final List<JsonEnabledColumnConfig> columns) {
columns.forEach(this.delegate::addColumn);
this.configMap = this.getColumns().stream()
.collect(Collectors.toMap(ColumnConfig::getName, Function.identity()));
}
@Override
public SqlStatement[] generateStatements(Database database) {
if (database instanceof PostgresDatabase) {
for (AddColumnConfig config : delegate.getColumns()) {
String columnType = config.getType();
// if postgres, change JSON type to JSONB before generating the statements as JSONB is more efficient.
if (columnType.equalsIgnoreCase("JSON")) {
config.setType("JSONB");
}
}
}
// AddColumnChange always produces an AddColumnStatement in the first position of the returned array.
AddColumnStatement delegateStatement = (AddColumnStatement) Arrays.stream(this.delegate.generateStatements(database))
.findFirst().get();
// convert the regular AddColumnStatements into GeneratedColumnStatements, adding the extension properties.
if (!delegateStatement.isMultiple()) {
// single statement - convert it directly.
JsonEnabledColumnConfig config = configMap.get(delegateStatement.getColumnName());
if (config != null) {
return new SqlStatement[] {new GeneratedColumnStatement(delegateStatement, config.getJsonColumn(),
config.getJsonProperty())};
}
}
else {
// multiple statement - convert all sub-statements.
List<GeneratedColumnStatement> generatedColumnStatements = delegateStatement.getColumns().stream()
.filter(c -> configMap.containsKey(c.getColumnName()))
.map(c -> new GeneratedColumnStatement(c, configMap.get(c.getColumnName()).getJsonColumn(),
configMap.get(c.getColumnName()).getJsonProperty()))
.collect(Collectors.toList());
// add all GeneratedColumnStatements into a composite statement and return the composite.
return new SqlStatement[]{new GeneratedColumnStatement(generatedColumnStatements)};
}
return new SqlStatement[0];
}
@Override
protected Change[] createInverses() {
return this.delegate.createInverses();
}
@Override
public ChangeStatus checkStatus(Database database) {
return delegate.checkStatus(database);
}
@Override
public String getConfirmationMessage() {
return delegate.getConfirmationMessage();
}
@Override
public String getSerializedObjectNamespace() {
return GENERIC_CHANGELOG_EXTENSION_NAMESPACE;
}
@Override
public ValidationErrors validate(Database database) {
ValidationErrors validationErrors = new ValidationErrors();
validationErrors.checkRequiredField("columns", this.delegate.getColumns());
// validate each generated column.
this.delegate.getColumns().stream().map(JsonEnabledColumnConfig.class::cast).forEach(
config -> {
if (config.isAutoIncrement() != null && config.isAutoIncrement()) {
validationErrors.addError("Generated column " + config.getName() + " cannot be auto-incremented");
} else if (config.getValueObject() != null) {
validationErrors.addError("Generated column " + config.getName() + " cannot be configured with a value");
} else if (config.getDefaultValueObject() != null) {
validationErrors.addError("Generated column " + config.getName() + " cannot be configured with a default value");
}
// we can expand this check if we decide to allow other types of generated columns in the future - for now
// ensure the column can be properly generated from a json property stored on a json column.
validationErrors.checkRequiredField("jsonColumn", config.getJsonColumn());
validationErrors.checkRequiredField("jsonProperty", config.getJsonProperty());
});
validationErrors.addAll(super.validate(database));
return validationErrors;
}
/**
* Simple extension that makes protected methods public so they can be accessed as a delegate.
*/
private static class ExtendedAddColumnChange extends AddColumnChange {
@Override
public Change[] createInverses() {
return super.createInverses();
}
}
}

View file

@ -0,0 +1,96 @@
/*
* Copyright 2021 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.models.map.storage.jpa.liquibase.extension;
import java.util.ArrayList;
import java.util.List;
import liquibase.database.Database;
import liquibase.database.core.PostgresDatabase;
import liquibase.sql.Sql;
import liquibase.sql.UnparsedSql;
import liquibase.sqlgenerator.SqlGenerator;
import liquibase.sqlgenerator.core.AddColumnGenerator;
import liquibase.statement.core.AddColumnStatement;
/**
* A {@link SqlGenerator} implementation that supports {@link GeneratedColumnStatement}s. It generates the SQL required
* to add a column whose values are generated from a property of a JSON file stored in one of the table columns.
*
* @author <a href="mailto:sguilhen@redhat.com">Stefan Guilhen</a>
*/
public class GeneratedColumnSqlGenerator extends AddColumnGenerator {
/**
* Override the priority. This is needed because {@link GeneratedColumnStatement} is a subtype of {@link AddColumnStatement}
* and is thus a match for the standard column generators. By increasing the priority we ensure this is processed before
* the other generators.
*
* @return this generator's priority.
*/
@Override
public int getPriority() {
return SqlGenerator.PRIORITY_DEFAULT + 1;
}
/**
* Implement {@link #supports(AddColumnStatement, Database)} to return {@code true} only if the statement type is an instance
* of {@link GeneratedColumnStatement}.
* </p>
* This is needed because this generator is a sub-class of {@link AddColumnGenerator} and is thus registered as being
* able to handle statements of type {@link AddColumnStatement}. Due to the increased priority, this generator ends up
* being selected to handle standard {@code addColumn} changes, which is not desirable. By returning {@code true} only
* when the statement is a {@link GeneratedColumnStatement} we ensure this implementation is selected only when a generated
* column is being added, allowing liquibase to continue iterating through the chain of generators in order to select the
* right generator to handle the standard {@code addColumn} changes.
*
* @param statement the {@link liquibase.statement.SqlStatement} to be processed.
* @param database a reference to the database.
* @return {@code true} if an only if the statement is a {@link GeneratedColumnStatement}; {@code false} otherwise.
*/
@Override
public boolean supports(AddColumnStatement statement, Database database) {
// use this implementation for generated columns only.
return statement instanceof GeneratedColumnStatement;
}
@Override
protected Sql[] generateSingleColumn(final AddColumnStatement statement, final Database database) {
StringBuilder sqlBuilder = new StringBuilder();
sqlBuilder.append(super.generateSingleColumBaseSQL(statement, database));
sqlBuilder.append(super.generateSingleColumnSQL(statement, database));
this.handleGeneratedColumn((GeneratedColumnStatement) statement, database, sqlBuilder);
List<Sql> returnSql = new ArrayList<>();
returnSql.add(new UnparsedSql(sqlBuilder.toString(), super.getAffectedColumn(statement)));
super.addUniqueConstrantStatements(statement, database, returnSql);
super.addForeignKeyStatements(statement, database, returnSql);
return returnSql.toArray(new Sql[0]);
}
protected void handleGeneratedColumn(final GeneratedColumnStatement statement, final Database database, final StringBuilder sqlBuilder) {
if (database instanceof PostgresDatabase) {
// assemble the GENERATED ALWAYS AS section of the query using the json property selection function.
sqlBuilder.append(" GENERATED ALWAYS AS ((").append(statement.getJsonColumn()).append("->>'").append(statement.getJsonProperty())
.append("')::").append(statement.getColumnType()).append(") stored");
}
}
}

View file

@ -0,0 +1,65 @@
/*
* Copyright 2021 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.models.map.storage.jpa.liquibase.extension;
import java.util.List;
import liquibase.statement.ColumnConstraint;
import liquibase.statement.core.AddColumnStatement;
/**
* A {@link liquibase.statement.SqlStatement} that extends the standard {@link AddColumnStatement} to include properties
* to identify the JSON column and JSON property that are to be used to generated the values for the column being added.
*
* @author <a href="mailto:sguilhen@redhat.com">Stefan Guilhen</a>
*/
public class GeneratedColumnStatement extends AddColumnStatement {
private String jsonColumn;
private String jsonProperty;
public GeneratedColumnStatement(final AddColumnStatement statement, final String jsonColumn, final String jsonProperty) {
super(statement.getCatalogName(), statement.getSchemaName(), statement.getTableName(), statement.getColumnName(),
statement.getColumnType(), statement.getDefaultValue(), statement.getRemarks(),
statement.getConstraints().toArray(new ColumnConstraint[0]));
this.jsonColumn = jsonColumn;
this.jsonProperty = jsonProperty;
}
public GeneratedColumnStatement(final List<GeneratedColumnStatement> statements) {
super(statements.toArray(new GeneratedColumnStatement[0]));
}
/**
* Obtains the name of the column that holds JSON files.
*
* @return the name of the JSON column.
*/
public String getJsonColumn() {
return this.jsonColumn;
}
/**
* Obtains the name of the property in the JSON file whose value is to be used as the generated value for the new column.
*
* @return the name of the JSON property.
*/
public String getJsonProperty() {
return this.jsonProperty;
}
}

View file

@ -0,0 +1,42 @@
/*
* Copyright 2021 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.models.map.storage.jpa.liquibase.extension;
import liquibase.database.Database;
import liquibase.database.core.PostgresDatabase;
import liquibase.datatype.DataTypeInfo;
import liquibase.datatype.DatabaseDataType;
import liquibase.datatype.LiquibaseDataType;
/**
* A {@link LiquibaseDataType} to handle the JSON column type.
*
* @author <a href="mailto:sguilhen@redhat.com">Stefan Guilhen</a>
*/
@DataTypeInfo(name="json", minParameters = 0, maxParameters = 1, priority = LiquibaseDataType.PRIORITY_DEFAULT + 1)
public class JsonDataType extends LiquibaseDataType {
@Override
public DatabaseDataType toDatabaseDataType(Database database) {
if (database instanceof PostgresDatabase) {
// on Postgres switch the columns of type JSON to JSONB as JSONB is a more efficient type to handle JSON contents.
return new DatabaseDataType("JSONB", super.getParameters());
}
return super.toDatabaseDataType(database);
}
}

View file

@ -0,0 +1,81 @@
/*
* Copyright 2021 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.models.map.storage.jpa.liquibase.extension;
import liquibase.change.AddColumnConfig;
import liquibase.parser.core.ParsedNode;
import liquibase.parser.core.ParsedNodeException;
import liquibase.resource.ResourceAccessor;
/**
* A {@link liquibase.change.ColumnConfig} extension that contains attributes to specify a JSON column and the property
* to be selected from the JSON file.
* </p>
* This config is used by extensions that need to operated on data stored in JSON columns.
*
* @author <a href="mailto:sguilhen@redhat.com">Stefan Guilhen</a>
*/
public class JsonEnabledColumnConfig extends AddColumnConfig {
private String jsonColumn;
private String jsonProperty;
/**
* Obtains the name of the column that contains JSON files.
*
* @return the JSON column name.
*/
public String getJsonColumn() {
return this.jsonColumn;
}
/**
* Sets the name of the column that contains JSON files.
*
* @param jsonColumn the name of the JSON column.
*/
public void setJsonColumn(final String jsonColumn) {
this.jsonColumn = jsonColumn;
}
/**
* Obtains the name of the property inside the JSON file.
*
* @return the name of the JSON property.
*/
public String getJsonProperty() {
return this.jsonProperty;
}
/**
* Sets the name of the property inside the JSON file.
*
* @param jsonProperty the name of the JSON property.
*/
public void setJsonProperty(final String jsonProperty) {
this.jsonProperty = jsonProperty;
}
@Override
public void load(ParsedNode parsedNode, ResourceAccessor resourceAccessor) throws ParsedNodeException {
// load the standard column attributs and then load the JSON attributes.
super.load(parsedNode, resourceAccessor);
this.jsonColumn = parsedNode.getChildValue(null, "jsonColumn", String.class);
this.jsonProperty = parsedNode.getChildValue(null, "jsonProperty", String.class);
}
}

View file

@ -17,37 +17,58 @@ limitations under the License.
-->
<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.1.xsd">
<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.1.xsd
http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd">
<!-- format of id of changeSet: clients-${JpaClientMapStorage.SUPPORTED_VERSION} -->
<changeSet author="keycloak" id="clients-1">
<sql>
create table client (
id uuid primary key not null,
entityVersion integer generated always as ((metadata->>'entityVersion')::int) stored,
realmId varchar(36) generated always as (metadata->>'fRealmId') stored,
clientId varchar(255) generated always as (metadata->>'fClientId') stored,
protocol varchar(36) generated always as (metadata->>'fProtocol') stored,
enabled boolean generated always as ((metadata->>'fEnabled')::boolean) stored,
metadata jsonb
);
create index client_entityVersion on client(entityVersion);
create index client_realmId_clientId on client(realmId, clientId);
create index client_scopeMappings on client using gin ((metadata->'fScopeMappings') jsonb_path_ops);
create table client_attribute (
id uuid primary key not null,
fk_client uuid references client(id) on delete cascade,
name varchar(255),
value text
);
create index client_attr_fk_client on client_attribute(fk_client);
create index client_attr_name_value on client_attribute(name, (value::varchar(250)));
</sql>
<createTable tableName="client">
<column name="id" type="UUID">
<constraints primaryKey="true" nullable="false"/>
</column>
<column name="metadata" type="json"/>
</createTable>
<ext:addGeneratedColumn tableName="client">
<ext:column name="entityversion" type="INTEGER" jsonColumn="metadata" jsonProperty="entityVersion"/>
<ext:column name="realmid" type="VARCHAR(36)" jsonColumn="metadata" jsonProperty="fRealmId"/>
<ext:column name="clientid" type="VARCHAR(255)" jsonColumn="metadata" jsonProperty="fClientId"/>
<ext:column name="protocol" type="VARCHAR(36)" jsonColumn="metadata" jsonProperty="fProtocol"/>
<ext:column name="enabled" type="BOOLEAN" jsonColumn="metadata" jsonProperty="fEnabled"/>
</ext:addGeneratedColumn>
<createIndex tableName="client" indexName="client_entityVersion">
<column name="entityversion"/>
</createIndex>
<createIndex tableName="client" indexName="client_realmId_clientId">
<column name="realmid"/>
<column name="clientid"/>
</createIndex>
<ext:createJsonIndex tableName="client" indexName="client_scopeMappings">
<ext:column jsonColumn="metadata" jsonProperty="fScopeMappings"/>
</ext:createJsonIndex>
<createTable tableName="client_attribute">
<column name="id" type="UUID">
<constraints primaryKey="true" nullable="false"/>
</column>
<column name="fk_client" type="UUID">
<constraints foreignKeyName="client_attr_fk_client_fkey" references="client(id)" deleteCascade="true"/>
</column>
<column name="name" type="VARCHAR(255)"/>
<column name="value" type="text"/>
</createTable>
<createIndex tableName="client_attribute" indexName="client_attr_fk_client">
<column name="fk_client"/>
</createIndex>
<createIndex tableName="client_attribute" indexName="client_attr_name_value">
<column name="name"/>
<column name="VALUE(255)" valueComputed="VALUE(255)"/>
</createIndex>
<modifySql dbms="postgresql">
<replace replace="VALUE(255)" with="(value::varchar(250))"/>
</modifySql>
</changeSet>
</databaseChangeLog>

View file

@ -0,0 +1,18 @@
#
# Copyright 2016 Red Hat, Inc. and/or its affiliates
# and other contributors as indicated by the @author tags.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
org.keycloak.models.map.storage.jpa.liquibase.DefaultLiquibaseConnectionProviderFactory

View file

@ -15,4 +15,4 @@
# limitations under the License.
#
org.keycloak.models.map.storage.jpa.updater.liquibase.MapJpaLiquibaseUpdaterProviderFactory
org.keycloak.models.map.storage.jpa.liquibase.MapJpaLiquibaseUpdaterProviderFactory

View file

@ -16,3 +16,4 @@
#
org.keycloak.models.map.storage.jpa.updater.MapJpaUpdaterSpi
org.keycloak.models.map.storage.jpa.liquibase.MapLiquibaseConnectionSpi