diff --git a/export-import/export-import-api/src/main/java/org/keycloak/exportimport/ExportImportConfig.java b/export-import/export-import-api/src/main/java/org/keycloak/exportimport/ExportImportConfig.java index a91cc3178aa..3ce6336436c 100644 --- a/export-import/export-import-api/src/main/java/org/keycloak/exportimport/ExportImportConfig.java +++ b/export-import/export-import-api/src/main/java/org/keycloak/exportimport/ExportImportConfig.java @@ -24,7 +24,11 @@ public class ExportImportConfig { // used for "singleFile" provider public static final String FILE = PREFIX + "file"; - // Number of users per file used in "dir" and "zip" providers. -1 means adding users to same file with realm. 0 means adding to separate file with unlimited page number + // How to export users when realm export is requested for "dir" and "zip" provider + public static final String USERS_EXPORT_STRATEGY = PREFIX + "usersExportStrategy"; + public static final UsersExportStrategy DEFAULT_USERS_EXPORT_STRATEGY = UsersExportStrategy.DIFFERENT_FILES; + + // Number of users per file used in "dir" and "zip" providers. Used if usersExportStrategy is DIFFERENT_FILES public static final String USERS_PER_FILE = PREFIX + "usersPerFile"; public static final Integer DEFAULT_USERS_PER_FILE = 5000; @@ -92,6 +96,15 @@ public class ExportImportConfig { System.setProperty(FILE, file); } + public static UsersExportStrategy getUsersExportStrategy() { + String usersExportStrategy = System.getProperty(USERS_EXPORT_STRATEGY, DEFAULT_USERS_EXPORT_STRATEGY.toString()); + return Enum.valueOf(UsersExportStrategy.class, usersExportStrategy); + } + + public static void setUsersExportStrategy(UsersExportStrategy usersExportStrategy) { + System.setProperty(USERS_EXPORT_STRATEGY, usersExportStrategy.toString()); + } + public static Integer getUsersPerFile() { String usersPerFile = System.getProperty(USERS_PER_FILE, String.valueOf(DEFAULT_USERS_PER_FILE)); return Integer.parseInt(usersPerFile.trim()); diff --git a/export-import/export-import-api/src/main/java/org/keycloak/exportimport/UsersExportStrategy.java b/export-import/export-import-api/src/main/java/org/keycloak/exportimport/UsersExportStrategy.java new file mode 100644 index 00000000000..c1df98902b9 --- /dev/null +++ b/export-import/export-import-api/src/main/java/org/keycloak/exportimport/UsersExportStrategy.java @@ -0,0 +1,11 @@ +package org.keycloak.exportimport; + +/** + * @author Marek Posolda + */ +public enum UsersExportStrategy { + SKIP, // Exporting of users will be skipped completely + REALM_FILE, // All users will be exported to same file with realm (So file like "foo-realm.json" with both realm data and users) + SAME_FILE, // All users will be exported to same file but different than realm (So file like "foo-realm.json" with realm data and "foo-users.json" with users) + DIFFERENT_FILES // Users will be exported into more different files according to maximum number of users per file +} diff --git a/export-import/export-import-api/src/main/java/org/keycloak/exportimport/util/ImportUtils.java b/export-import/export-import-api/src/main/java/org/keycloak/exportimport/util/ImportUtils.java index b1d37ba5e68..1425b361e2d 100755 --- a/export-import/export-import-api/src/main/java/org/keycloak/exportimport/util/ImportUtils.java +++ b/export-import/export-import-api/src/main/java/org/keycloak/exportimport/util/ImportUtils.java @@ -51,8 +51,12 @@ public class ImportUtils { } else { logger.infof("Realm '%s' already exists. Removing it before import", realmName); if (Config.getAdminRealm().equals(realm.getId())) { - realm.setMasterAdminApp(null); + // Delete all masterAdmin apps due to foreign key constraints + for (RealmModel currRealm : model.getRealms()) { + currRealm.setMasterAdminApp(null); + } } + // TODO: For migration between versions, it should be possible to delete just realm but keep it's users model.removeRealm(realm.getId()); } } @@ -138,9 +142,21 @@ public class ImportUtils { if (parser.getCurrentToken() == JsonToken.START_ARRAY) { // Case with more realms in stream parser.nextToken(); + + List realmReps = new ArrayList(); while (parser.getCurrentToken() == JsonToken.START_OBJECT) { RealmRepresentation realmRep = parser.readValueAs(RealmRepresentation.class); parser.nextToken(); + + // Ensure that master realm is imported first + if (Config.getAdminRealm().equals(realmRep.getRealm())) { + realmReps.add(0, realmRep); + } else { + realmReps.add(realmRep); + } + } + + for (RealmRepresentation realmRep : realmReps) { importRealm(session, realmRep, strategy); } } else if (parser.getCurrentToken() == JsonToken.START_OBJECT) { diff --git a/export-import/export-import-api/src/main/java/org/keycloak/exportimport/util/MultipleStepsExportProvider.java b/export-import/export-import-api/src/main/java/org/keycloak/exportimport/util/MultipleStepsExportProvider.java index bf07427e99c..9395ceb5a9b 100755 --- a/export-import/export-import-api/src/main/java/org/keycloak/exportimport/util/MultipleStepsExportProvider.java +++ b/export-import/export-import-api/src/main/java/org/keycloak/exportimport/util/MultipleStepsExportProvider.java @@ -6,6 +6,7 @@ import java.util.List; import org.jboss.logging.Logger; import org.keycloak.exportimport.ExportImportConfig; import org.keycloak.exportimport.ExportProvider; +import org.keycloak.exportimport.UsersExportStrategy; import org.keycloak.models.KeycloakSession; import org.keycloak.models.KeycloakSessionFactory; import org.keycloak.models.RealmModel; @@ -23,7 +24,6 @@ public abstract class MultipleStepsExportProvider implements ExportProvider { public void exportModel(KeycloakSessionFactory factory) throws IOException { final RealmsHolder holder = new RealmsHolder(); - // Import users into same file with realm ExportImportUtils.runJobInTransaction(factory, new ExportImportJob() { @Override @@ -41,33 +41,34 @@ public abstract class MultipleStepsExportProvider implements ExportProvider { @Override public void exportRealm(KeycloakSessionFactory factory, final String realmName) throws IOException { + final UsersExportStrategy usersExportStrategy = ExportImportConfig.getUsersExportStrategy(); final int usersPerFile = ExportImportConfig.getUsersPerFile(); final UsersHolder usersHolder = new UsersHolder(); - final boolean exportUsersIntoSameFile = usersPerFile < 0; + final boolean exportUsersIntoRealmFile = usersExportStrategy == UsersExportStrategy.REALM_FILE; ExportImportUtils.runJobInTransaction(factory, new ExportImportJob() { @Override public void run(KeycloakSession session) throws IOException { RealmModel realm = session.realms().getRealmByName(realmName); - RealmRepresentation rep = ExportUtils.exportRealm(session, realm, exportUsersIntoSameFile); + RealmRepresentation rep = ExportUtils.exportRealm(session, realm, exportUsersIntoRealmFile); writeRealm(realmName + "-realm.json", rep); logger.info("Realm '" + realmName + "' - data exported"); // Count total number of users - if (!exportUsersIntoSameFile) { + if (!exportUsersIntoRealmFile) { usersHolder.totalCount = session.users().getUsersCount(realm); } } }); - if (!exportUsersIntoSameFile) { - + if (usersExportStrategy != UsersExportStrategy.SKIP && !exportUsersIntoRealmFile) { + // We need to export users now usersHolder.currentPageStart = 0; - // usersPerFile==0 means exporting all users into single file (but separate to realm) - final int countPerPage = usersPerFile == 0 ? usersHolder.totalCount : usersPerFile; + // usersExportStrategy==SAME_FILE means exporting all users into single file (but separate to realm) + final int countPerPage = (usersExportStrategy == UsersExportStrategy.SAME_FILE) ? usersHolder.totalCount : usersPerFile; while (usersHolder.currentPageStart < usersHolder.totalCount) { if (usersHolder.currentPageStart + countPerPage < usersHolder.totalCount) { diff --git a/export-import/export-import-dir/src/main/java/org/keycloak/exportimport/dir/DirImportProvider.java b/export-import/export-import-dir/src/main/java/org/keycloak/exportimport/dir/DirImportProvider.java index ef29af9459a..512004d3819 100644 --- a/export-import/export-import-dir/src/main/java/org/keycloak/exportimport/dir/DirImportProvider.java +++ b/export-import/export-import-dir/src/main/java/org/keycloak/exportimport/dir/DirImportProvider.java @@ -4,8 +4,14 @@ import java.io.File; import java.io.FileInputStream; import java.io.FilenameFilter; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; import org.jboss.logging.Logger; +import org.keycloak.Config; import org.keycloak.exportimport.ImportProvider; import org.keycloak.exportimport.Strategy; import org.keycloak.exportimport.util.ExportImportJob; @@ -54,11 +60,21 @@ public class DirImportProvider implements ImportProvider { } }); + List realmNames = new ArrayList(); for (File file : realmFiles) { String fileName = file.getName(); - // Parse "foo" from "foo-realm.json" String realmName = fileName.substring(0, fileName.length() - 11); + + // Ensure that master realm is imported first + if (Config.getAdminRealm().equals(realmName)) { + realmNames.add(0, realmName); + } else { + realmNames.add(realmName); + } + } + + for (String realmName : realmNames) { importRealm(factory, realmName, strategy); } } diff --git a/export-import/export-import-zip/src/main/java/org/keycloak/exportimport/zip/ZipImportProvider.java b/export-import/export-import-zip/src/main/java/org/keycloak/exportimport/zip/ZipImportProvider.java index 20d3dc75bfd..6970e87899f 100644 --- a/export-import/export-import-zip/src/main/java/org/keycloak/exportimport/zip/ZipImportProvider.java +++ b/export-import/export-import-zip/src/main/java/org/keycloak/exportimport/zip/ZipImportProvider.java @@ -4,6 +4,8 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.zip.DataFormatException; import de.idyl.winzipaes.AesZipFileDecrypter; @@ -11,6 +13,7 @@ import de.idyl.winzipaes.impl.AESDecrypter; import de.idyl.winzipaes.impl.AESDecrypterBC; import de.idyl.winzipaes.impl.ExtZipEntry; import org.jboss.logging.Logger; +import org.keycloak.Config; import org.keycloak.exportimport.ImportProvider; import org.keycloak.exportimport.Strategy; import org.keycloak.exportimport.util.ExportImportJob; @@ -48,14 +51,25 @@ public class ZipImportProvider implements ImportProvider { @Override public void importModel(KeycloakSessionFactory factory, Strategy strategy) throws IOException { + List realmNames = new ArrayList(); for (ExtZipEntry entry : this.decrypter.getEntryList()) { String entryName = entry.getName(); if (entryName.endsWith("-realm.json")) { // Parse "foo" from "foo-realm.json" String realmName = entryName.substring(0, entryName.length() - 11); - importRealm(factory, realmName, strategy); + + // Ensure that master realm is imported first + if (Config.getAdminRealm().equals(realmName)) { + realmNames.add(0, realmName); + } else { + realmNames.add(realmName); + } } } + + for (String realmName : realmNames) { + importRealm(factory, realmName, strategy); + } } @Override diff --git a/model/api/src/main/java/org/keycloak/models/utils/RepresentationToModel.java b/model/api/src/main/java/org/keycloak/models/utils/RepresentationToModel.java index 35fe9c9bfd8..f20cb9521b5 100755 --- a/model/api/src/main/java/org/keycloak/models/utils/RepresentationToModel.java +++ b/model/api/src/main/java/org/keycloak/models/utils/RepresentationToModel.java @@ -118,10 +118,7 @@ public class RepresentationToModel { } for (RoleRepresentation roleRep : entry.getValue()) { // Application role may already exists (for example if it is defaultRole) - RoleModel role = app.getRole(roleRep.getName()); - if (role == null) { - role = app.addRole(roleRep.getName()); - } + RoleModel role = roleRep.getId()!=null ? app.addRole(roleRep.getId(), roleRep.getName()) : app.addRole(roleRep.getName()); role.setDescription(roleRep.getDescription()); } } @@ -147,12 +144,21 @@ public class RepresentationToModel { } } - + // Setup realm default roles if (rep.getDefaultRoles() != null) { for (String roleString : rep.getDefaultRoles()) { newRealm.addDefaultRole(roleString.trim()); } } + // Setup application default roles + if (rep.getApplications() != null) { + for (ApplicationRepresentation resourceRep : rep.getApplications()) { + if (resourceRep.getDefaultRoles() != null) { + ApplicationModel appModel = newRealm.getApplicationByName(resourceRep.getName()); + appModel.updateDefaultRoles(resourceRep.getDefaultRoles()); + } + } + } if (rep.getOauthClients() != null) { createOAuthClients(rep, newRealm); @@ -336,7 +342,7 @@ public class RepresentationToModel { private static Map createApplications(RealmRepresentation rep, RealmModel realm) { Map appMap = new HashMap(); for (ApplicationRepresentation resourceRep : rep.getApplications()) { - ApplicationModel app = createApplication(realm, resourceRep); + ApplicationModel app = createApplication(realm, resourceRep, false); appMap.put(app.getName(), app); } return appMap; @@ -349,7 +355,7 @@ public class RepresentationToModel { * @param resourceRep * @return */ - public static ApplicationModel createApplication(RealmModel realm, ApplicationRepresentation resourceRep) { + public static ApplicationModel createApplication(RealmModel realm, ApplicationRepresentation resourceRep, boolean addDefaultRoles) { logger.debug("************ CREATE APPLICATION: {0}" + resourceRep.getName()); ApplicationModel applicationModel = resourceRep.getId()!=null ? realm.addApplication(resourceRep.getId(), resourceRep.getName()) : realm.addApplication(resourceRep.getName()); if (resourceRep.isEnabled() != null) applicationModel.setEnabled(resourceRep.isEnabled()); @@ -403,7 +409,7 @@ public class RepresentationToModel { } } - if (resourceRep.getDefaultRoles() != null) { + if (addDefaultRoles && resourceRep.getDefaultRoles() != null) { applicationModel.updateDefaultRoles(resourceRep.getDefaultRoles()); } diff --git a/model/tests/src/test/java/org/keycloak/model/test/ApplicationModelTest.java b/model/tests/src/test/java/org/keycloak/model/test/ApplicationModelTest.java index ed62c3731a6..66e36e2db37 100755 --- a/model/tests/src/test/java/org/keycloak/model/test/ApplicationModelTest.java +++ b/model/tests/src/test/java/org/keycloak/model/test/ApplicationModelTest.java @@ -61,7 +61,7 @@ public class ApplicationModelTest extends AbstractModelTest { ApplicationRepresentation representation = ModelToRepresentation.toRepresentation(application); RealmModel realm = realmManager.createRealm("copy"); - ApplicationModel copy = RepresentationToModel.createApplication(realm, representation); + ApplicationModel copy = RepresentationToModel.createApplication(realm, representation, true); assertEquals(application, copy); } diff --git a/services/src/main/java/org/keycloak/services/resources/admin/ApplicationsResource.java b/services/src/main/java/org/keycloak/services/resources/admin/ApplicationsResource.java index 68926ece4b5..610e7726a3a 100755 --- a/services/src/main/java/org/keycloak/services/resources/admin/ApplicationsResource.java +++ b/services/src/main/java/org/keycloak/services/resources/admin/ApplicationsResource.java @@ -87,7 +87,7 @@ public class ApplicationsResource { auth.requireManage(); try { - ApplicationModel applicationModel = RepresentationToModel.createApplication(realm, rep); + ApplicationModel applicationModel = RepresentationToModel.createApplication(realm, rep, true); return Response.created(uriInfo.getAbsolutePathBuilder().path(applicationModel.getName()).build()).build(); } catch (ModelDuplicateException e) { return Flows.errors().exists("Application " + rep.getName() + " already exists"); diff --git a/testsuite/integration/src/test/java/org/keycloak/testsuite/exportimport/ExportImportTest.java b/testsuite/integration/src/test/java/org/keycloak/testsuite/exportimport/ExportImportTest.java index 6919c045229..746a87a4188 100755 --- a/testsuite/integration/src/test/java/org/keycloak/testsuite/exportimport/ExportImportTest.java +++ b/testsuite/integration/src/test/java/org/keycloak/testsuite/exportimport/ExportImportTest.java @@ -81,7 +81,7 @@ public class ExportImportTest { @Override protected void after() { if (previousMongoClearOnStartup != null) { - System.setProperty(MONGO_CLEAR_ON_STARTUP_PROP_NAME, "false"); + System.setProperty(MONGO_CLEAR_ON_STARTUP_PROP_NAME, previousMongoClearOnStartup); } else { System.getProperties().remove(MONGO_CLEAR_ON_STARTUP_PROP_NAME); } @@ -128,7 +128,7 @@ public class ExportImportTest { .around(mongoRule) .around(keycloakRule); - //@Test + @Test public void testDirFullExportImport() throws Throwable { ExportImportConfig.setProvider(DirExportProviderFactory.PROVIDER_ID); String targetDirPath = getExportImportTestDirectory() + File.separator + "dirExport";