Refactoring. Adding callbacks to auto-update registry & auto-index post migrations. Updates to Unit Tests and code to use those callbacks. Cleanup of Ant build process based on changes

dspace-api/src/main/java/org/dspace/administer/MetadataImporter.java

@@ -26,6 +26,8 @@ import org.dspace.content.MetadataField;
 import org.dspace.content.MetadataSchema;
 import org.dspace.content.NonUniqueMetadataException;
 import org.dspace.core.Context;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import org.w3c.dom.Document;
 import org.w3c.dom.Node;
@@ -56,6 +58,9 @@ import org.xml.sax.SAXException;
  */
 public class MetadataImporter
 {
+    /** logging category */
+    private static final Logger log = LoggerFactory.getLogger(MetadataImporter.class);
+
 	/**
 	 * main method for reading user input from the command line
 	 */
@@ -96,10 +101,14 @@ public class MetadataImporter
     public static void loadRegistry(String file, boolean forceUpdate)
     	throws SQLException, IOException, TransformerException, ParserConfigurationException, 
     		AuthorizeException, SAXException, NonUniqueMetadataException, RegistryImportException
+    {
+        Context context = null;
+
+        try
         {
             // create a context
-        Context context = new Context();
+            context = new Context();
-        context.setIgnoreAuthorization(true);
+            context.turnOffAuthorisationSystem();
 
             // read the XML
             Document document = RegistryImporter.loadXML(file);
@@ -124,8 +133,16 @@ public class MetadataImporter
                 loadType(context, n);
             }
 
+            context.restoreAuthSystemState();
             context.complete();
         }
+        finally
+        {
+           // Clean up our context, if it still exists & it was never completed
+            if(context!=null && context.isValid())
+                context.abort();
+        }
+    }
     
     /**
      * Process a node in the metadata registry XML file.  If the
@@ -155,24 +172,22 @@ public class MetadataImporter
             throw new RegistryImportException("Namespace of schema must be supplied");
         }
 
-        System.out.print("Registering Schema: " + name + " - " + namespace + " ... ");
-        
         // check to see if the schema already exists
         MetadataSchema s = MetadataSchema.find(context, name);
         
         if (s == null)
         {
             // Schema does not exist - create
+            log.info("Registering Schema " + name + " (" + namespace + ")");
             MetadataSchema schema = new MetadataSchema(namespace, name);
             schema.create(context);
-            System.out.println("created");
         }
         else
         {
             // Schema exists - if it's the same namespace, allow the type imports to continue
             if (s.getNamespace().equals(namespace))
             {
-                System.out.println("already exists, skipping to type import");
+                // This schema already exists with this namespace, skipping it
                 return;
             }
             
@@ -180,19 +195,13 @@ public class MetadataImporter
             if (updateExisting)
             {
                 // Update the existing schema namespace and continue to type import
+                log.info("Updating Schema " + name + ": New namespace " + namespace);
                 s.setNamespace(namespace);
                 s.update(context);
-                System.out.println("namespace updated (" + name + " = " + namespace + ")");
             }
             else
             {
-                // Don't update the existing namespace - abort abort abort
+                throw new RegistryImportException("Schema " + name + " already registered with different namespace " + namespace + ". Rerun with 'update' option enabled if you wish to update this schema.");
-                System.out.println("schema exists, but with different namespace");
-                System.out.println("was: " + s.getNamespace());
-                System.out.println("xml: " + namespace);
-                System.out.println("aborting - use -u to force the update");
-                
-                throw new RegistryImportException("schema already registered with different namespace - use -u to update");
             }
         }
         
@@ -225,30 +234,33 @@ public class MetadataImporter
             schema = MetadataSchema.DC_SCHEMA;
         }
 
-        System.out.print("Registering Metadata: " + schema + "." + element + "." + qualifier + " ... ");
         
         // Find the matching schema object
         MetadataSchema schemaObj = MetadataSchema.find(context, schema);
         
         if (schemaObj == null)
         {
-            throw new RegistryImportException("Schema '" + schema + "' is not registered");
+            throw new RegistryImportException("Schema '" + schema + "' is not registered and does not exist.");
         }
         
         MetadataField mf = MetadataField.findByElement(context, schemaObj.getSchemaID(), element, qualifier);
         if (mf != null)
         {
-            System.out.println("already exists, skipping");
+            // Metadata field already exists, skipping it
             return;
         }
         
+        // Actually create this metadata field as it doesn't yet exist
+        String fieldName = schema + "." + element + "." + qualifier;
+        if(qualifier==null)
+            fieldName = schema + "." + element;
+        log.info("Registering metadata field " + fieldName);
         MetadataField field = new MetadataField();
         field.setSchemaID(schemaObj.getSchemaID());
         field.setElement(element);
         field.setQualifier(qualifier);
         field.setScopeNote(scopeNote);
         field.create(context);
-        System.out.println("created");
     }
     
     /**

dspace-api/src/main/java/org/dspace/administer/RegistryLoader.java

@@ -57,7 +57,7 @@ public class RegistryLoader
     public static void main(String[] argv) throws Exception
     {
         String usage = "Usage: " + RegistryLoader.class.getName()
-                + " (-bitstream | -dc) registry-file.xml";
+                + " (-bitstream | -metadata) registry-file.xml";
 
         Context context = null;
 
@@ -67,22 +67,24 @@ public class RegistryLoader
 
             // Can't update registries anonymously, so we need to turn off
             // authorisation
-            context.setIgnoreAuthorization(true);
+            context.turnOffAuthorisationSystem();
 
             // Work out what we're loading
             if (argv[0].equalsIgnoreCase("-bitstream"))
             {
                 RegistryLoader.loadBitstreamFormats(context, argv[1]);
             }
-            else if (argv[0].equalsIgnoreCase("-dc"))
+            else if (argv[0].equalsIgnoreCase("-metadata"))
             {
-                loadDublinCoreTypes(context, argv[1]);
+                // Call MetadataImporter, as it handles Metadata schema updates
+                MetadataImporter.loadRegistry(argv[1], true);
             }
             else
             {
                 System.err.println(usage);
             }
 
+            // Commit changes and close Context
             context.complete();
 
             System.exit(0);
@@ -91,11 +93,6 @@ public class RegistryLoader
         {
             System.err.println(usage);
 
-            if (context != null)
-            {
-                context.abort();
-            }
-
             System.exit(1);
         }
         catch (Exception e)
@@ -103,14 +100,15 @@ public class RegistryLoader
             log.fatal(LogManager.getHeader(context, "error_loading_registries",
                     ""), e);
 
-            if (context != null)
-            {
-                context.abort();
-            }
-
             System.err.println("Error: \n - " + e.getMessage());
             System.exit(1);
         }
+        finally
+        {
+            // Clean up our context, if it still exists & it was never completed
+            if(context!=null && context.isValid())
+                context.abort();
+        }
     }
 
     /**
@@ -168,6 +166,12 @@ public class RegistryLoader
 
         String[] extensions = getRepeatedElementData(node, "extension");
 
+        // Check if this format already exists in our registry (by mime type)
+        BitstreamFormat exists = BitstreamFormat.findByMIMEType(context, mimeType);
+        
+        // If it doesn't exist, create it..otherwise skip it.
+        if(exists==null)
+        {
             // Create the format object
             BitstreamFormat format = BitstreamFormat.create(context);
 
@@ -182,110 +186,6 @@ public class RegistryLoader
             // Write to database
             format.update();
         }
-
-    /**
-     * Load Dublin Core types
-     * 
-     * @param context
-     *            DSpace context object
-     * @param filename
-     *            the filename of the XML file to load
-     * @throws NonUniqueMetadataException
-     */
-    public static void loadDublinCoreTypes(Context context, String filename)
-            throws SQLException, IOException, ParserConfigurationException,
-            SAXException, TransformerException, AuthorizeException,
-            NonUniqueMetadataException
-    {
-        Document document = loadXML(filename);
-
-        // Get the nodes corresponding to schemas
-        NodeList schemaNodes = XPathAPI.selectNodeList(document,
-                "/dspace-dc-types/dc-schema");
-
-        // Add each schema
-        for (int i = 0; i < schemaNodes.getLength(); i++)
-        {
-            Node n = schemaNodes.item(i);
-            loadMDSchema(context, n);
-        }
-        
-        // Get the nodes corresponding to fields
-        NodeList typeNodes = XPathAPI.selectNodeList(document,
-                "/dspace-dc-types/dc-type");
-
-        // Add each one as a new field to the schema
-        for (int i = 0; i < typeNodes.getLength(); i++)
-        {
-            Node n = typeNodes.item(i);
-            loadDCType(context, n);
-        }
-
-        log.info(LogManager.getHeader(context, "load_dublin_core_types",
-                "number_loaded=" + typeNodes.getLength()));
-    }
-
-    /**
-     * Load Dublin Core Schemas
-     * 
-     * @param context
-     * @param node
-     */
-    private static void loadMDSchema(Context context, Node node) 
-    		throws TransformerException, SQLException, AuthorizeException, 
-    		NonUniqueMetadataException
-    {
-    	// Get the values
-        String shortname = getElementData(node, "name");
-        String namespace = getElementData(node, "namespace");
-
-        // Check if the schema exists already
-        MetadataSchema schema = MetadataSchema.find(context, shortname);
-        if (schema == null)
-        {
-        	// If not create it.
-        	schema = new MetadataSchema();
-        	schema.setNamespace(namespace);
-        	schema.setName(shortname);
-        	schema.create(context);
-        }
-    }
-    
-    /**
-     * Process a node in the bitstream format registry XML file. The node must
-     * be a "bitstream-type" node
-     * 
-     * @param context
-     *            DSpace context object
-     * @param node
-     *            the node in the DOM tree
-     * @throws NonUniqueMetadataException
-     */
-    private static void loadDCType(Context context, Node node)
-            throws SQLException, IOException, TransformerException,
-            AuthorizeException, NonUniqueMetadataException
-    {
-        // Get the values
-        String schema = getElementData(node, "schema");
-        String element = getElementData(node, "element");
-        String qualifier = getElementData(node, "qualifier");
-        String scopeNote = getElementData(node, "scope_note");
-
-        // If the schema is not provided default to DC
-        if (schema == null)
-        {
-            schema = MetadataSchema.DC_SCHEMA;
-        }
-
-        // Find the matching schema object
-        MetadataSchema schemaObj = MetadataSchema.find(context, schema);
-        
-        MetadataField field = new MetadataField();
-        field.setSchemaID(schemaObj.getSchemaID());
-        field.setElement(element);
-        field.setQualifier(qualifier);
-        field.setScopeNote(scopeNote);
-        field.create(context);
     }
 
     // ===================== XML Utility Methods =========================

dspace-api/src/main/java/org/dspace/content/DSpaceObject.java

@@ -1278,7 +1278,7 @@ public abstract class DSpaceObject
 
                             if (field == null)
                             {
-                                log.error("Loading item - cannot find metadata field " + fieldID);
+                                log.error("Loading item - cannot find metadata field " + fieldID + " for resourceType=" + resourceTypeId + " and resourceId=" + resourceId);
                             }
                             else
                             {

dspace-api/src/main/java/org/dspace/discovery/SolrServiceImpl.java

@@ -154,7 +154,7 @@ public class SolrServiceImpl implements SearchService, IndexingService {
 
                     solr.query(solrQuery);                
                 } catch (SolrServerException e) {
-                    log.error("Error while initialinging solr server", e);
+                    log.error("Error while initializing solr server", e);
                 }
             }
             else

dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseManager.java

@@ -77,13 +77,6 @@ public class DatabaseManager
     /** Name to use for the pool */
     private static String poolName = "dspacepool";
     
-    /** Database Status Flags for Flyway setup. Fresh Install vs. pre-4.0 vs */
-    private static final int STATUS_PRE_4_0 = -1;
-    private static final int STATUS_FRESH_INSTALL = 0;
-    private static final int STATUS_NO_FLYWAY = 1;
-    private static final int STATUS_FLYWAY = 2;
-    
-
     /**
      * This regular expression is used to perform sanity checks
      * on database names (i.e. tables and columns).
@@ -1585,10 +1578,10 @@ public class DatabaseManager
             }
             log.info("DBMS driver version is '{}'", meta.getDatabaseProductVersion());
             
-            // FINALLY, ensure database scheme is up-to-date. If not, upgrade/migrate database.
+            // FINALLY, ensure database schema is up-to-date.
-            // (NOTE: This needs to run LAST as it may need some of the initialized
+            // If not, upgrade/migrate database. (NOTE: This needs to run LAST
-            // variables set above)
+            // as it may need some of the initialized variables set above)
-            initializeDatabase(connection);
+            DatabaseUtils.updateDatabase(dataSource, connection);
 
             connection.close();
             initialized = true;
@@ -1607,137 +1600,6 @@ public class DatabaseManager
         }
     }
     
-    /**
-     * Ensures the current database is up-to-date with regards
-     * to the latest DSpace DB schema. If the scheme is not up-to-date,
-     * then any necessary database migrations are performed.
-     * 
-     * @param connection
-     *      Database connection
-     */
-    private static synchronized void initializeDatabase(Connection connection) 
-            throws IOException, SQLException
-    {
-        // Get the name of the Schema that the DSpace Database is using
-        String schema = ConfigurationManager.getProperty("db.schema");
-        if(StringUtils.isBlank(schema)){
-            schema = null;
-        }
-        
-        // Initialize Flyway DB API (http://flywaydb.org/), used to perform DB migrations
-        Flyway flyway = new Flyway();
-        flyway.setDataSource(dataSource);
-        flyway.setEncoding("UTF-8");
-    
-        // Migration scripts are based on DBMS Keyword (see full path below)
-        String scriptFolder = dbms_keyword;
-        
-        // Set location where Flyway will load DB scripts from (based on DB Type)
-        // e.g. [dspace.dir]/etc/[dbtype]/
-        String scriptPath = ConfigurationManager.getProperty("dspace.dir") +
-                            System.getProperty("file.separator") + "etc" +
-                            System.getProperty("file.separator") + "migrations" +
-                            System.getProperty("file.separator") + scriptFolder;
-        
-        log.info("Loading Flyway DB scripts from " + scriptPath + " and Package 'org.dspace.storage.rdbms.migration.*'");
-        flyway.setLocations("filesystem:" + scriptPath, "classpath:org.dspace.storage.rdbms.migration");
-        
-        // Get our Database migration status, so we know what to tell Flyway to do
-        int status = getDbMigrationStatus(schema, connection, flyway);
-        
-        // If we have a pre-4.0 Database, we need to exit immediately. There's nothing we can do here
-        if(status==STATUS_PRE_4_0)
-            throw new SQLException("CANNOT AUTOUPGRADE DSPACE DATABASE, AS IT DOES NOT LOOK TO BE A VALID DSPACE 4.0 DATABASE. " +
-                        "Please manually upgrade your database to DSpace 4.0 compatibility.");
-        // If this is a fresh install
-        else if (status==STATUS_FRESH_INSTALL)
-        {
-            // Just let Flyway initialize our database
-            flyway.init();
-        }
-        // If we have a valid 4.0 database, but haven't initialized Flyway on it
-        else if (status == STATUS_NO_FLYWAY)
-        {
-            // Initialize the Flyway database table.
-            // We are hardcoding the schema version to 4.0 because this should ONLY
-            // be encountered on a 4.0 database. After 4.0, all databases should 
-            // already have Flyway initialized.
-            // (NOTE: Flyway will also create the db.schema, if it doesn't exist)
-            flyway.setInitVersion("4.0");
-            flyway.setInitDescription("Initial DSpace 4.0 database schema");
-            flyway.init();
-        }
-           
-        // Determine pending Database migrations
-        MigrationInfo[] pending = flyway.info().pending();
-        
-        // Log info about pending migrations
-        if (pending!=null && pending.length>0) 
-        {   
-            log.info("Pending DSpace database schema migrations:");
-            for (MigrationInfo info : pending)
-            {
-                log.info("\t" + info.getVersion() + " " + info.getDescription() + " " + info.getType() + " " + info.getState());
-            }
-        }
-        else
-            log.info("DSpace database schema is up to date.");
-
-        // Ensure database is on the latest version of the DSpace schema
-        flyway.migrate();
-    }
-    
-    /**
-     * Determine the migration status of our Database
-     * so that we are able to properly migrate it to the latest schema
-     * via Flyway
-     * 
-     * @param schema
-     *          Name of the Schema being used by the DSpace database
-     * @param connection
-     *          Current Database Connection
-     * @param flyway
-     *          Our Flyway settings
-     * @return status flag 
-     */
-    private static int getDbMigrationStatus(String schema, Connection connection, Flyway flyway)
-            throws SQLException
-    {
-        // Get information about our database. We'll use this to determine DB status.
-        DatabaseMetaData meta = connection.getMetaData();
-       
-        // First, is this a "fresh_install"?  Check for an "item" table.
-        ResultSet tables = meta.getTables(null, schema, "item", null);
-        if (!tables.next()) 
-        {
-            tables.close();
-            // No "item" table, this is a fresh install of DSpace
-            return STATUS_FRESH_INSTALL;
-        }
-  
-        // Second, is this DSpace DB Schema compatible with 4.0?  Check for a "Webapp" table (which was added in 4.0)
-        // TODO: If the "Webapp" table is ever removed, then WE NEED TO CHANGE THIS CHECK.
-        tables = meta.getTables(null, schema, "Webapp", null);
-        if (!tables.next()) 
-        {
-            tables.close();
-            // No "Webapp" table, so this must be a pre-4.0 database
-            return STATUS_PRE_4_0;
-        }
-        
-        // Finally, Check if the necessary Flyway table ("schema_version") exists in this database
-        tables = meta.getTables(null, schema, flyway.getTable(), null);
-        if (!tables.next()) 
-        {
-            tables.close();
-            // No Flyway table, so we need to get Flyway initialized in this database
-            return STATUS_NO_FLYWAY;
-        }
-        
-        // IF we get here, we have 4.0 or above compatible database and Flyway is already installed
-        return STATUS_FLYWAY;
-    }
-    
     /**
      * What is the name of our DBMS?
      *

dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseRegistryUpdater.java

@@ -0,0 +1,202 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE and NOTICE files at the root of the source
+ * tree and available online at
+ *
+ * http://www.dspace.org/license/
+ */
+package org.dspace.storage.rdbms;
+
+import java.io.File;
+import java.sql.Connection;
+import org.dspace.administer.MetadataImporter;
+import org.dspace.administer.RegistryLoader;
+import org.dspace.core.ConfigurationManager;
+import org.dspace.core.Context;
+import org.flywaydb.core.api.MigrationInfo;
+import org.flywaydb.core.api.callback.FlywayCallback;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This is a FlywayCallback class which automatically updates the
+ * Metadata Schema Registry and Bitstream Formats Registries BEFORE
+ * any Database migration occurs.
+ * <P>
+ * The reason this runs BEFORE a migration is to ensure that any new
+ * metadata fields are FIRST added to our registries, so that the 
+ * migrations can make use of those new metadata fields, etc.
+ * <P>
+ * However, there is one exception. If this is a "fresh install" of DSpace, 
+ * we'll need to wait until the necessary database tables are created. In
+ * that scenario we will load registries AFTER the initial migrations.
+ * 
+ * @author Tim Donohue
+ */
+public class DatabaseRegistryUpdater implements FlywayCallback
+{
+     /** logging category */
+    private static final Logger log = LoggerFactory.getLogger(DatabaseRegistryUpdater.class);
+    
+    // Whether or not this is a fresh install of DSpace
+    // This determines whether to update registries PRE or POST migration
+    private boolean freshInstall = false;
+    
+    /**
+     * Method to actually update our registries from latest configs
+     */
+    private void updateRegistries()
+    {
+        Context context = null;
+        try
+        {
+            context = new Context();
+            context.turnOffAuthorisationSystem();
+            
+            String base = ConfigurationManager.getProperty("dspace.dir")
+                            + File.separator + "config" + File.separator
+                            + "registries" + File.separator;
+            
+            // Load updates to Bitstream format registry (if any)
+            log.info("Updating Bitstream Format Registry based on " + base + "bitstream-formats.xml");
+            RegistryLoader.loadBitstreamFormats(context, base + "bitstream-formats.xml");
+
+            // Load updates to Metadata schema registries (if any)
+            log.info("Updating Metadata Registries based on metadata type configs in " + base);
+            MetadataImporter.loadRegistry(base + "dublin-core-types.xml", true);
+            MetadataImporter.loadRegistry(base + "dcterms-types.xml", true);
+            MetadataImporter.loadRegistry(base + "eperson-types.xml", true);
+            MetadataImporter.loadRegistry(base + "sword-metadata.xml", true);
+
+            // Check if XML Workflow is enabled in workflow.cfg
+            if (ConfigurationManager.getProperty("workflow", "workflow.framework").equals("xmlworkflow"))
+            {
+                // If so, load in the workflow metadata types as well
+                MetadataImporter.loadRegistry(base + "workflow-types.xml", true);
+            }
+            
+            context.restoreAuthSystemState();
+            // Commit changes and close context
+            context.complete();
+            log.info("All Bitstream Format Regitry and Metadata Registry updates were completed.");
+        }
+        catch(Exception e)
+        {
+            log.error("Error attempting to update Bitstream Format and/or Metadata Registries", e);
+        }
+        finally
+        {
+            // Clean up our context, if it still exists & it was never completed
+            if(context!=null && context.isValid())
+                context.abort();
+        }
+    }
+    
+    
+    @Override
+    public void afterClean(Connection connection)
+    {
+        // do nothing
+    }
+    
+    @Override
+    public void afterEachMigrate(Connection connection, MigrationInfo info)
+    {
+        // do nothing
+    }
+    
+    @Override
+    public void afterInfo(Connection connection)
+    {
+        // do nothing
+    }
+    
+    @Override
+    public void afterInit(Connection connection)
+    {
+        // do nothing
+    }
+    
+    @Override
+    public void afterMigrate(Connection connection)
+    {
+        // If this is a fresh install, we must update registries AFTER the
+        // initial migrations (since the registry tables won't exist until the
+        // initial migrations are performed)
+        if(freshInstall)
+        {
+            updateRegistries();
+            freshInstall = false;
+        }
+    }
+    
+    @Override
+    public void afterRepair(Connection connection)
+    {
+        // do nothing
+    }
+    
+    @Override
+    public void afterValidate(Connection connection)
+    {
+        // do nothing
+    }
+    
+    @Override
+    public void beforeClean(Connection connection)
+    {
+        // do nothing
+    }
+    
+    @Override
+    public void beforeEachMigrate(Connection connection, MigrationInfo info)
+    {
+        // do nothing
+    }
+    
+    @Override
+    public void beforeInfo(Connection connection)
+    {
+        // do nothing
+    }
+    
+    @Override
+    public void beforeInit(Connection connection)
+    {
+        // do nothing
+    }
+    
+    @Override
+    public void beforeMigrate(Connection connection)
+    {
+        // Check if our MetadataSchemaRegistry table exists yet. 
+        // If it does NOT, then this is a fresh install & we'll need to 
+        // updateRegistries() AFTER migration 
+        if(DatabaseUtils.tableExists(connection, "MetadataSchemaRegistry"))
+        {
+            // Ensure registries are updated BEFORE a database migration (upgrade)
+            // We need to ensure any new metadata fields are added before running
+            // migrations, just in case the migrations need to utilize those new fields
+            updateRegistries();
+        }
+        else
+        {
+            // this is a fresh install, need to migrate first in order to create
+            // the registry tables.
+            freshInstall = true;
+        }
+    }
+    
+    @Override
+    public void beforeRepair(Connection connection)
+    {
+        // do nothing
+    }
+    
+    @Override
+    public void beforeValidate(Connection connection)
+    {
+        // do nothing
+    }
+    
+}

dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseReindexer.java

@@ -0,0 +1,208 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE and NOTICE files at the root of the source
+ * tree and available online at
+ *
+ * http://www.dspace.org/license/
+ */
+package org.dspace.storage.rdbms;
+
+import java.sql.Connection;
+import java.util.Arrays;
+import java.util.List;
+import org.dspace.browse.IndexBrowse;
+import org.dspace.core.Context;
+import org.dspace.discovery.IndexingService;
+import org.dspace.discovery.SearchServiceException;
+import org.dspace.search.DSIndexer;
+import org.dspace.services.ConfigurationService;
+import org.dspace.utils.DSpace;
+import org.flywaydb.core.api.MigrationInfo;
+import org.flywaydb.core.api.callback.FlywayCallback;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This is a FlywayCallback class which automatically reindexes Database
+ * contents into your search/browse engine of choice.
+ * <P>
+ * Reindexing is performed AFTER any database migration or repair. This
+ * ensures that your search/browse indexes are auto-updated post-upgrade.
+ * 
+ * @author Tim Donohue
+ */
+public class DatabaseReindexer implements FlywayCallback
+{
+    /** logging category */
+    private static final Logger log = LoggerFactory.getLogger(DatabaseReindexer.class);
+    
+    /**
+     * Method to actually reindex all database contents. This method is "smart"
+     * in that it determines which indexing consumer(s) you have enabled, 
+     * and then ensures each is reindexed appropriately.
+     */
+    private void reindex()
+    {
+        Context context = null;
+        try
+        {
+            context = new Context();
+
+            // What indexing consumer(s) are configured in this DSpace?
+            ConfigurationService config = new DSpace().getConfigurationService();
+            String consumers = config.getPropertyAsType("event.dispatcher.default.consumers", ""); // Avoid null pointer
+            List<String> consumerList = Arrays.asList(consumers.split("\\s*,\\s*"));
+
+            // If Discovery indexing is enabled
+            if (consumerList.contains("discovery"))
+            {
+                log.info("Reindexing all content in Discovery search and browse engine");
+                try
+                {
+                    // Reindex Discovery (just clean & update index)
+                    DSpace dspace = new DSpace();
+                    IndexingService indexer = dspace.getServiceManager().getServiceByName(IndexingService.class.getName(),IndexingService.class);
+                    indexer.cleanIndex(true);
+                    indexer.updateIndex(context, true);
+                }
+                catch(SearchServiceException sse)
+                {
+                    log.warn("Unable to reindex content in Discovery search and browse engine. You will need to reindex manually.", sse);
+                }
+            }
+            
+            // If Lucene indexing is enabled
+            if (consumerList.contains("search"))
+            {
+                log.info("Reindexing all content in Lucene search engine");
+                // Clean and update Lucene index
+                DSIndexer.cleanIndex(context);
+                DSIndexer.updateIndex(context, true);
+            }
+            
+            // If traditional DBMS browse indexing is enabled
+            if (consumerList.contains("browse"))
+            {
+                log.info("Reindexing all content in DBMS Browse tables");
+                // Rebuild browse tables to perform a full index 
+                // (recreating tables as needed)
+                IndexBrowse indexer = new IndexBrowse(context);
+                indexer.setRebuild(true);
+                indexer.setExecute(true);
+                indexer.initBrowse();
+                // Since the browse index is in the DB, we must commit & close context
+                context.complete();
+            }
+            else
+            {
+                log.info("Checking for (and cleaning up) unused DBMS Browse tables");
+                // If traditional browse tables are not being used, 
+                // then clean them up from database (they can always be recreated later)
+                IndexBrowse indexer = new IndexBrowse(context);
+                indexer.clearDatabase();
+                // Commit changes to browse tables & close context
+                context.complete();
+            }
+            
+            log.info("Reindexing is complete");
+        }
+        catch(Exception e)
+        {
+            log.error("Error attempting to reindex all contents for search/browse", e);
+        }
+        finally
+        {
+            // Clean up our context, if it still exists & it was never completed
+            if(context!=null && context.isValid())
+                context.abort();
+        }
+    }
+    
+    
+    @Override
+    public void afterClean(Connection connection)
+    {
+        // do nothing
+    }
+    
+    @Override
+    public void afterEachMigrate(Connection connection, MigrationInfo info)
+    {
+        // do nothing
+    }
+    
+    @Override
+    public void afterInfo(Connection connection)
+    {
+        // do nothing
+    }
+    
+    @Override
+    public void afterInit(Connection connection)
+    {
+        // do nothing
+    }
+    
+    @Override
+    public void afterMigrate(Connection connection)
+    {
+        // Reindex after a database migration (upgrade)
+        reindex();
+    }
+    
+    @Override
+    public void afterRepair(Connection connection)
+    {
+        // Reindex after a database repair
+        reindex();
+    }
+    
+    @Override
+    public void afterValidate(Connection connection)
+    {
+        // do nothing
+    }
+    
+    @Override
+    public void beforeClean(Connection connection)
+    {
+        // do nothing
+    }
+    
+    @Override
+    public void beforeEachMigrate(Connection connection, MigrationInfo info)
+    {
+        // do nothing
+    }
+    
+    @Override
+    public void beforeInfo(Connection connection)
+    {
+        // do nothing
+    }
+    
+    @Override
+    public void beforeInit(Connection connection)
+    {
+        // do nothing
+    }
+    
+    @Override
+    public void beforeMigrate(Connection connection)
+    {
+        // do nothing
+    }
+    
+    @Override
+    public void beforeRepair(Connection connection)
+    {
+        // do nothing
+    }
+    
+    @Override
+    public void beforeValidate(Connection connection)
+    {
+        // do nothing
+    }
+    
+}

dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseUtils.java

@@ -0,0 +1,332 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE and NOTICE files at the root of the source
+ * tree and available online at
+ *
+ * http://www.dspace.org/license/
+ */
+package org.dspace.storage.rdbms;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import javax.sql.DataSource;
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
+import org.dspace.core.ConfigurationManager;
+import org.flywaydb.core.Flyway;
+import org.flywaydb.core.api.MigrationInfo;
+import org.flywaydb.core.internal.info.MigrationInfoDumper;
+
+/**
+ * Utility class used to manage the Database. This class is used by the
+ * DatabaseManager to initialize/upgrade/migrate the Database. It can also
+ * be called via the commandline as necessary to get information about
+ * the database.
+ * <p>
+ * Currently, we use Flyway DB (http://flywaydb.org/) for database management.
+ * 
+ * @see org.dspace.storage.rdbms.DatabaseManager
+ * @author Tim Donohue
+ */
+public class DatabaseUtils 
+{
+    /** log4j category */
+    private static final Logger log = Logger.getLogger(DatabaseUtils.class);
+    
+    // Our Flyway DB object (initialized by setupFlyway())
+    private static Flyway flywaydb;
+    
+    /** Database Status Flags for Flyway setup. Fresh Install vs. pre-4.0 vs */
+    private static final int STATUS_PRE_4_0 = -1;
+    private static final int STATUS_FRESH_INSTALL = 0;
+    private static final int STATUS_NO_FLYWAY = 1;
+    private static final int STATUS_FLYWAY = 2;
+    
+    /**
+     * Commandline tools for managing database changes, etc.
+     * @param argv 
+     */
+    public static void main(String[] argv)
+    {  
+        // Usage checks
+        if (argv.length != 1)
+        {
+            System.out.println("\nDatabase action argument is missing.");
+            System.out.println("Valid actions include: 'info', 'migrate', 'repair' or 'clean'");
+            System.exit(1);
+        }
+
+        try
+        {
+            // This is just to ensure DatabaseManager.initialize() gets called!
+            String dbName = DatabaseManager.getDbName();
+            System.out.println("Initialized connection to " + dbName + " database");
+
+            // Setup Flyway against our database
+            Flyway flyway = setupFlyway(DatabaseManager.getDataSource());
+            
+            if(argv[0].equalsIgnoreCase("info"))
+            {
+                // Get basic Database info
+                Connection connection = DatabaseManager.getConnection();
+                DatabaseMetaData meta = connection.getMetaData();
+                System.out.println("\nDatabase: " + meta.getDatabaseProductName() + " version " + meta.getDatabaseProductVersion());
+                System.out.println("Database Driver: " + meta.getDriverName() + " version " + meta.getDriverVersion());
+                
+                // Get info table from Flyway
+                System.out.println("\n" + MigrationInfoDumper.dumpToAsciiTable(flyway.info().all()));
+            }
+            else if(argv[0].equalsIgnoreCase("migrate"))
+            {
+                System.out.println("Migrating database to latest version... (Check logs for details)");
+                flyway.migrate();
+            }
+            else if(argv[0].equalsIgnoreCase("repair"))
+            {
+                System.out.println("Attempting to repair database via FlywayDB... (Check logs for details)");
+                flyway.repair();
+            }
+            else if(argv[0].equalsIgnoreCase("clean"))
+            {
+                BufferedReader input = new BufferedReader(new InputStreamReader(System.in));
+                System.out.println("If you continue, ALL DATA IN YOUR DATABASE WILL BE DELETED. \n");
+                System.out.println("There is no turning back from this action. You should backup your database before continuing. \n");
+                System.out.println("Are you ready to destroy your entire database? [y/n]: ");
+                String choiceString = input.readLine();
+                input.close();
+                
+                if (choiceString.equalsIgnoreCase("y"))
+                {
+                    System.out.println("Scrubbing database clean... (Check logs for details)");
+                    flyway.clean();
+                }
+            }
+            else
+            {
+                System.out.println("\nDatabase action " + argv[0] + " is not valid.");
+                System.out.println("Valid actions include: 'info', 'migrate', 'repair' or 'clean'");
+            }
+
+            System.exit(0);
+        }
+        catch (Exception e)
+        {
+            System.err.println("Caught exception:");
+            e.printStackTrace();
+            System.exit(1);
+        }
+    }
+    
+    
+    
+    /**
+     * Setup/Initialize the Flyway API to run against our DSpace database
+     * and point at our migration scripts.
+     * 
+     * @param datasource
+     *      DataSource object initialized by DatabaseManager
+     * @return initialized Flyway object
+     */
+    private static Flyway setupFlyway(DataSource datasource)
+    {
+        if (flywaydb==null)
+        {
+            // Initialize Flyway DB API (http://flywaydb.org/), used to perform DB migrations
+            flywaydb = new Flyway();
+            flywaydb.setDataSource(datasource);
+            flywaydb.setEncoding("UTF-8");
+    
+            // Migration scripts are based on DBMS Keyword (see full path below)
+            String scriptFolder = DatabaseManager.getDbKeyword();
+
+            // Set location where Flyway will load DB scripts from (based on DB Type)
+            // e.g. [dspace.dir]/etc/[dbtype]/
+            String scriptPath = ConfigurationManager.getProperty("dspace.dir") +
+                                System.getProperty("file.separator") + "etc" +
+                                System.getProperty("file.separator") + "migrations" +
+                                System.getProperty("file.separator") + scriptFolder;
+
+            // Flyway will look in "scriptPath" for SQL migrations AND
+            // in 'org.dspace.storage.rdbms.migration.*' for Java migrations
+            log.info("Loading Flyway DB migrations from " + scriptPath + " and Package 'org.dspace.storage.rdbms.migration.*'");
+            flywaydb.setLocations("filesystem:" + scriptPath, "classpath:org.dspace.storage.rdbms.migration");
+            
+            // Set flyway callbacks (i.e. classes which are called post-DB migration and similar)
+            // In this situation, we have a Registry Updater that runs PRE-migration
+            // and a Reindexer that runs POST-migration
+            flywaydb.setCallbacks(new DatabaseRegistryUpdater(), new DatabaseReindexer());
+        }
+        
+        return flywaydb;
+    }
+    
+    /**
+     * Ensures the current database is up-to-date with regards
+     * to the latest DSpace DB schema. If the scheme is not up-to-date,
+     * then any necessary database migrations are performed.
+     * <P>
+     * FlywayDB (http://flywaydb.org/) is used to perform database migrations.
+     * If a Flyway DB migration fails it will be rolled back to the last
+     * successful migration, and any errors will be logged.
+     * 
+     * @param datasource
+     *      DataSource object (retrieved from DatabaseManager())
+     * @param connection
+     *      Database connection
+     * @throws SQLException
+     *      If database cannot be upgraded.
+     */
+    protected static synchronized void updateDatabase(DataSource datasource, Connection connection) 
+            throws SQLException
+    {
+        // Setup Flyway against our database
+        Flyway flyway = setupFlyway(datasource);
+        
+        // Get our Database migration status, so we know what to tell Flyway to do
+        int status = getDbMigrationStatus(connection, flyway);
+        
+        // If we have a pre-4.0 Database, we need to exit immediately. There's nothing we can do here
+        if(status==STATUS_PRE_4_0)
+            throw new SQLException("CANNOT AUTOUPGRADE DSPACE DATABASE, AS IT DOES NOT LOOK TO BE A VALID DSPACE 4.0 DATABASE. " +
+                        "Please manually upgrade your database to DSpace 4.0 compatibility.");
+        // If this is a fresh install
+        else if (status==STATUS_FRESH_INSTALL)
+        {
+            // Initialize the Flyway database table
+            flyway.init();
+        }
+        // If we have a valid 4.0 database, but haven't initialized Flyway on it
+        else if (status == STATUS_NO_FLYWAY)
+        {
+            // Initialize the Flyway database table.
+            // We are hardcoding the schema version to 4.0 because this should ONLY
+            // be encountered on a 4.0 database. After 4.0, all databases should 
+            // already have Flyway initialized.
+            // (NOTE: Flyway will also create the db.schema, if it doesn't exist)
+            flyway.setInitVersion("4.0");
+            flyway.setInitDescription("Initial DSpace 4.0 database schema");
+            flyway.init();
+        }
+           
+        // Determine pending Database migrations
+        MigrationInfo[] pending = flyway.info().pending();
+        
+        // Log info about pending migrations
+        if (pending!=null && pending.length>0) 
+        {   
+            log.info("Pending DSpace database schema migrations:");
+            for (MigrationInfo info : pending)
+            {
+                log.info("\t" + info.getVersion() + " " + info.getDescription() + " " + info.getType() + " " + info.getState());
+            }
+        }
+        else
+            log.info("DSpace database schema is up to date.");
+
+        // Run all pending Flyway migrations to ensure the DSpace Database is up to date
+        flyway.migrate();
+    }
+    
+    /**
+     * Determine the migration status of our Database
+     * so that we are able to properly migrate it to the latest schema
+     * via Flyway
+     * 
+     * @param connection
+     *          Current Database Connection
+     * @param flyway
+     *          Our Flyway settings
+     * @throws SQLException if DB status cannot be determined
+     * @return status flag 
+     */
+    private static int getDbMigrationStatus(Connection connection, Flyway flyway)
+            throws SQLException
+    {
+        // Get information about our database. We'll use this to determine DB status.
+        DatabaseMetaData meta = connection.getMetaData();
+       
+        // First, is this a "fresh_install"?  Check for an "item" table.
+        if(!tableExists(connection, "item"))
+        {
+            // No "item" table, this is a fresh install of DSpace
+            return STATUS_FRESH_INSTALL;
+        }
+  
+        // Second, is this DSpace DB Schema compatible with 4.0?  Check for a "Webapp" table (which was added in 4.0)
+        // TODO: If the "Webapp" table is ever removed, then WE NEED TO CHANGE THIS CHECK.
+        if(!tableExists(connection, "Webapp"))
+        {
+            // No "Webapp" table, so this must be a pre-4.0 database
+            return STATUS_PRE_4_0;
+        }
+        
+        // Finally, Check if the necessary Flyway table ("schema_version") exists in this database
+        if(!tableExists(connection, flyway.getTable()))
+        {
+            // No Flyway table, so we need to get Flyway initialized in this database
+            return STATUS_NO_FLYWAY;
+        }
+        
+        // IF we get here, we have 4.0 or above compatible database and Flyway is already installed
+        return STATUS_FLYWAY;
+    }
+    
+    
+    /**
+     * Determine if a particular database table exists in our database
+     * 
+     * @param connection
+     *          Current Database Connection
+     * @param tableName
+     *          The name of the table
+     * @return true if table of that name exists, false otherwise
+     */
+    public static boolean tableExists(Connection connection, String tableName)
+    {
+        // Get the name of the Schema that the DSpace Database is using
+        // (That way we can search the right schema for this table)
+        String schema = ConfigurationManager.getProperty("db.schema");
+        if(StringUtils.isBlank(schema)){
+            schema = null;
+        }
+        
+        boolean exists = false;        
+        ResultSet results = null;
+        
+        try
+        {
+            // Get information about our database.
+            DatabaseMetaData meta = connection.getMetaData();
+            
+            // Search for a table of the given name in our current schema
+            results = meta.getTables(null, schema, tableName, null);
+            if (results!=null && results.next()) 
+            {
+                exists = true;
+            }   
+        }
+        catch(SQLException e)
+        {
+            log.error("Error attempting to determine if table " + tableName + " exists", e);
+        }
+        finally
+        {
+            try
+            {
+                // ensure the ResultSet gets closed
+                if(results!=null && !results.isClosed())
+                    results.close();
+            }
+            catch(SQLException e)
+            {
+                // ignore it
+            }
+        }
+        
+        return exists;
+    }
+}

dspace-api/src/main/java/org/dspace/storage/rdbms/InitializeDatabase.java

@@ -1,95 +0,0 @@
-/**
- * The contents of this file are subject to the license and copyright
- * detailed in the LICENSE and NOTICE files at the root of the source
- * tree and available online at
- *
- * http://www.dspace.org/license/
- */
-package org.dspace.storage.rdbms;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.Locale;
-
-import org.apache.log4j.Logger;
-
-/**
- * Command-line executed class for initializing the DSpace database. This should
- * be invoked with a single argument, the filename of the database schema file.
- * 
- * @author Robert Tansley
- * @version $Revision$
- */
-public class InitializeDatabase
-{
-    /** log4j category */
-    private static final Logger log = Logger.getLogger(InitializeDatabase.class);
-
-    public static void main(String[] argv)
-    {
-      
-        // Usage checks
-        /*if (argv.length != 1)
-        {
-            log.warn("Schema file not specified");
-            System.exit(1);
-        }*/
-
-        log.info("Initializing Database");
-
-        try
-        {
-            // This is just to ensure DatabaseManager.initialize() gets called!
-            String dbKeyword = DatabaseManager.getDbKeyword();
-
-            log.info("Done initializing " + dbKeyword + " Database");
-            /*if("clean-database.sql".equals(argv[0]))
-            {
-               DatabaseManager.loadSql(getScript(argv[0]));
-            }
-            else
-            {
-               DatabaseManager.loadSql(getScript(argv[0]));
-            }*/
-            
-            System.exit(0);
-        }
-        catch (Exception e)
-        {
-            log.fatal("Caught exception:", e);
-            System.exit(1);
-        }
-    }
-
-    /**
-     * Attempt to get the named script, with the following rules:
-     * etc/<DBMS name>/<name>
-     * etc/<name>
-     * <name>
-     */
-    private static FileReader getScript(String name) throws FileNotFoundException, IOException
-    {
-        String dbName = DatabaseManager.getDbKeyword();
-
-        File myFile = null;
-        
-        if (dbName != null)
-        {
-            myFile = new File("etc/" + dbName + "/" + name);
-            if (myFile.exists())
-            {
-                return new FileReader(myFile.getCanonicalPath());
-            }
-        }
-        
-        myFile = new File("etc/" + name);
-        if (myFile.exists())
-        {
-            return new FileReader(myFile.getCanonicalPath());
-        }
-        
-        return new FileReader(name);
-    }
-}

dspace-api/src/test/java/org/dspace/AbstractUnitTest.java

@@ -9,39 +9,27 @@ package org.dspace;
 
 import static org.junit.Assert.fail;
 
-import java.io.File;
 import java.io.IOException;
 import java.net.URL;
 import java.sql.SQLException;
 import java.util.Properties;
 import java.util.TimeZone;
 
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.transform.TransformerException;
-
 import org.apache.log4j.Logger;
-import org.dspace.administer.MetadataImporter;
-import org.dspace.administer.RegistryImportException;
-import org.dspace.administer.RegistryLoader;
 import org.dspace.authorize.AuthorizeException;
-import org.dspace.content.MetadataField;
-import org.dspace.content.NonUniqueMetadataException;
 import org.dspace.core.ConfigurationManager;
 import org.dspace.core.Context;
 import org.dspace.core.I18nUtil;
-import org.dspace.discovery.IndexingService;
 import org.dspace.discovery.MockIndexEventConsumer;
 import org.dspace.eperson.EPerson;
 import org.dspace.eperson.Group;
 import org.dspace.servicemanager.DSpaceKernelImpl;
 import org.dspace.servicemanager.DSpaceKernelInit;
 import org.dspace.storage.rdbms.MockDatabaseManager;
-import org.dspace.utils.DSpace;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
-import org.xml.sax.SAXException;
 
 
 
@@ -73,13 +61,10 @@ public class AbstractUnitTest
     /**
      * EPerson mock object to use in the tests.
      */
-    protected static EPerson eperson;
+    protected EPerson eperson;
 
     protected static DSpaceKernelImpl kernelImpl;
 
-    // Whether the in-memory DB has been initiailzed for testing
-    protected static boolean dbInitialized = false;
-
     /** 
      * This method will be run before the first test as per @BeforeClass. It will
      * initialize resources required for the tests.
@@ -119,98 +104,13 @@ public class AbstractUnitTest
                 kernelImpl.start(ConfigurationManager.getProperty("dspace.dir"));
             }
             
-            // Applies/initializes our mock database by invoking its constructor:
+            // Applies/initializes our mock database by invoking its constructor
+            // (NOTE: This also initializes the DatabaseManager, which in turn
+            // calls DatabaseUtils to initialize the entire DB via Flyway)
             new MockDatabaseManager();
             
-            // Also initialize these mock classes for general use
+            // Initialize a mock indexer (which does nothing, since Solr isn't running)
             new MockIndexEventConsumer();
-
-            // Load the default registries. This assumes the temporary
-            // filesystem is working and the in-memory DB in place.
-            Context ctx = new Context();
-            ctx.turnOffAuthorisationSystem();
-
-            // We can't check via a boolean value (even static) as the class is
-            // destroyed by the JUnit classloader. We rely on a value that will
-            // always be in the database, if it has been initialized, to avoid
-            // doing the work twice.
-            if(MetadataField.find(ctx, 1) == null)
-            {
-                String base = ConfigurationManager.getProperty("dspace.dir")
-                        + File.separator + "config" + File.separator
-                        + "registries" + File.separator;
-
-                RegistryLoader.loadBitstreamFormats(ctx, base + "bitstream-formats.xml");
-                MetadataImporter.loadRegistry(base + "dublin-core-types.xml", true);
-                MetadataImporter.loadRegistry(base + "eperson-types.xml", true);
-                MetadataImporter.loadRegistry(base + "bitstream-formats.xml", true);
-                MetadataImporter.loadRegistry(base + "sword-metadata.xml", true);
-                ctx.commit();
-
-                //create eperson if required
-                eperson = EPerson.find(ctx, 1);
-                if(eperson == null)
-                {
-                    eperson = EPerson.create(ctx);
-                    eperson.setFirstName("first");
-                    eperson.setLastName("last");
-                    eperson.setEmail("test@email.com");
-                    eperson.setCanLogIn(true);
-                    eperson.setLanguage(I18nUtil.getDefaultLocale().getLanguage());
-                }
-
-                //Create search and browse indexes
-                DSpace dspace = new DSpace();
-                IndexingService indexer = dspace.getServiceManager().getServiceByName(IndexingService.class.getName(),IndexingService.class);
-                indexer.createIndex(ctx);
-                ctx.commit();
-                
-                // Nullify resources, so Junit will clean them up
-                dspace = null;
-                indexer = null;
-            }
-            Group.initDefaultGroupNames(ctx);
-            ctx.restoreAuthSystemState();
-            if(ctx.isValid())
-            {
-                ctx.complete();
-            }
-            ctx = null;    
-        } 
-        catch (RegistryImportException ex)
-        {
-            log.error("Error loading default data", ex);
-            fail("Error loading default data");
-        }
-        catch (NonUniqueMetadataException ex)
-        {
-            log.error("Error loading default data", ex);
-            fail("Error loading default data");
-        }
-        catch (ParserConfigurationException ex)
-        {
-            log.error("Error loading default data", ex);
-            fail("Error loading default data");
-        }
-        catch (SAXException ex)
-        {
-            log.error("Error loading default data", ex);
-            fail("Error loading default data");
-        }
-        catch (TransformerException ex)
-        {
-            log.error("Error loading default data", ex);
-            fail("Error loading default data");
-        }
-        catch (AuthorizeException ex)
-        {
-            log.error("Error loading default data", ex);
-            fail("Error loading default data");
-        }
-        catch (SQLException ex)
-        {
-            log.error("Error initializing the database", ex);
-            fail("Error initializing the database");
         } 
         catch (IOException ex)
         {
@@ -219,104 +119,6 @@ public class AbstractUnitTest
         }
     }
 
-    /**
-     * Copies one directory (And its contents) into another 
-     * 
-     * @param from Folder to copy
-     * @param to Destination
-     * @throws IOException There is an error while copying the content
-     */
-    /*
-    protected static void copyDir(File from, File to) throws IOException
-    {
-        if(!from.isDirectory() || !to.isDirectory())
-        {
-            throw new IOException("Both parameters must be directories. from is "+from.isDirectory()+", to is "+to.isDirectory());
-        }
-
-        File[] contents = from.listFiles();
-        for(File f: contents)
-        {
-            if(f.isFile())
-            {
-                File copy = new File(to.getAbsolutePath() + File.separator + f.getName());
-                copy.createNewFile();
-                copyFile(f, copy);
-            }
-            else if(f.isDirectory())
-            {
-                File copy = new File(to.getAbsolutePath() + File.separator + f.getName());
-                copy.mkdir();
-                copyDir(f, copy);
-            }
-        }
-    }
-     */
-
-    /**
-     * Removes the copies of the origin files from the destination folder. Used
-     * to remove the temporal copies of files done for testing
-     *
-     * @param from Folder to check
-     * @param to Destination from which to remove contents
-     * @throws IOException There is an error while copying the content
-     */
-    /*
-    protected static void deleteDir(File from, File to) throws IOException
-    {
-        if(!from.isDirectory() || !to.isDirectory())
-        {
-            throw new IOException("Both parameters must be directories. from is "+from.isDirectory()+", to is "+to.isDirectory());
-        }
-
-        File[] contents = from.listFiles();
-        for(File f: contents)
-        {
-            if(f.isFile())
-            {
-                File copy = new File(to.getAbsolutePath() + File.separator + f.getName());
-                if(copy.exists())
-                {
-                    copy.delete();
-                }
-                
-            }
-            else if(f.isDirectory())
-            {
-                File copy = new File(to.getAbsolutePath() + File.separator + f.getName());
-                if(copy.exists() && copy.listFiles().length > 0)
-                {
-                    deleteDir(f, copy);
-                }
-                copy.delete();
-            }
-        }
-    }
-     */
-
-    /**
-     * Copies one file into another
-     *
-     * @param from File to copy
-     * @param to Destination of copy
-     * @throws IOException There is an error while copying the content
-     */
-    /*
-    protected static void copyFile(File from, File to) throws IOException
-    {
-        if(!from.isFile() || !to.isFile())
-        {
-            throw new IOException("Both parameters must be files. from is "+from.isFile()+", to is "+to.isFile());
-        }
-
-        FileChannel in = (new FileInputStream(from)).getChannel();
-        FileChannel out = (new FileOutputStream(to)).getChannel();
-        in.transferTo(0, from.length(), out);
-        in.close();
-        out.close();
-    }
-     */
-
     /**
      * This method will be run before every test as per @Before. It will
      * initialize resources required for the tests.
@@ -329,11 +131,39 @@ public class AbstractUnitTest
     {        
         try
         {
-            //we start the context
+            //Start a new context
             context = new Context();
+            context.turnOffAuthorisationSystem();
+
+            //Find our global test EPerson account. If it doesn't exist, create it.
+            eperson = EPerson.findByEmail(context, "test@email.com");
+            if(eperson == null)
+            {
+                // This EPerson creation should only happen once (i.e. for first test run)
+                log.info("Creating initial EPerson (email=test@email.com) for Unit Tests");
+                eperson = EPerson.create(context);
+                eperson.setFirstName("first");
+                eperson.setLastName("last");
+                eperson.setEmail("test@email.com");
+                eperson.setCanLogIn(true);
+                eperson.setLanguage(I18nUtil.getDefaultLocale().getLanguage());
+                // actually save the eperson to unit testing DB
+                eperson.update();
+            }
+            // Set our global test EPerson as the current user in DSpace
             context.setCurrentUser(eperson);
+
+            // If our Anonymous/Administrator groups aren't initialized, initialize them as well
+            Group.initDefaultGroupNames(context);
+
+            context.restoreAuthSystemState();
             context.commit();
         }
+        catch (AuthorizeException ex)
+        {
+            log.error("Error creating initial eperson or default groups", ex);
+            fail("Error creating initial eperson or default groups in AbstractUnitTest init()");
+        }
         catch (SQLException ex) 
         {
             log.error(ex.getMessage(),ex);

dspace-api/src/test/java/org/dspace/storage/rdbms/MockDatabaseManager.java

@@ -43,12 +43,6 @@ public final class MockDatabaseManager
     // Set our logger to specify the Mock class, so we know which logs are from the "real" vs "mock" class
     private static final Logger log = Logger.getLogger(MockDatabaseManager.class);
 
-    // Is our DBMS Oracle-like? Determine this from the (initialized) DatabaseManager class
-    private static final boolean isOracle = DatabaseManager.isOracle();
-
-    // Get the database type keyword from the (initialized) DatabaseManager class
-    private static final String dbms_keyword = DatabaseManager.getDbKeyword();
-
     /**
      * Override/Mock the default "setConstraintDeferred()" method in order to
      * add some custom H2-specific code (look for the comments with "H2" in them).
@@ -65,7 +59,10 @@ public final class MockDatabaseManager
     public static void setConstraintDeferred(Invocation inv, Context context,
             String constraintName) throws SQLException
     {
-        if(dbms_keyword!=null && !dbms_keyword.equals(DatabaseManager.DBMS_H2))
+        // What type of database is this?
+        String databaseType = DatabaseManager.getDbKeyword();
+
+        if(databaseType!=null && !databaseType.equals(DatabaseManager.DBMS_H2))
         {
             // If we are unit testing with a non-H2 database, just proceed to
             // DatabaseManager method of the same name
@@ -119,7 +116,10 @@ public final class MockDatabaseManager
     public static void setConstraintImmediate(Invocation inv, Context context,
             String constraintName) throws SQLException
     {
-        if(dbms_keyword!=null && !dbms_keyword.equals(DatabaseManager.DBMS_H2))
+        // What type of database is this?
+        String databaseType = DatabaseManager.getDbKeyword();
+
+        if(databaseType!=null && !databaseType.equals(DatabaseManager.DBMS_H2))
         {
             // If we are unit testing with a non-H2 database, just proceed to
             // DatabaseManager method of the same name
@@ -175,7 +175,12 @@ public final class MockDatabaseManager
     @Mock
     static TableRow process(Invocation inv, ResultSet results, String table, List<String> pColumnNames) throws SQLException
     {
-        if(dbms_keyword!=null && !dbms_keyword.equals(DatabaseManager.DBMS_H2))
+        // What type of database is this?
+        String databaseType = DatabaseManager.getDbKeyword();
+        // Also, is it Oracle-like?
+        boolean isOracle = DatabaseManager.isOracle();
+
+        if(databaseType!=null && !databaseType.equals(DatabaseManager.DBMS_H2))
         {
             // If we are unit testing with a non-H2 database, just proceed to
             // DatabaseManager method of the same name

dspace/src/main/config/build.xml

@@ -26,7 +26,7 @@
 
 Common usage:
 
-  Fresh install, including database setup and registry loading:
+  Fresh install of DSpace:
     % ant fresh_install
 
   Update existing installation, leaving data and configuration intact:
@@ -136,14 +136,9 @@ Common usage:
         <echo message="init_configs    --> Write the configuration files to ${dspace.dir}/config" />
         <echo message="install_code    --> Install compiled code into ${dspace.dir}" />
         <echo message="" />
-        <echo message="fresh_install   --> Perform a fresh installation of the software, " />
+        <echo message="fresh_install   --> Perform a fresh installation of the software. " />
-        <echo message="                    including the databases &amp; config" />
-        <echo message="setup_database  --> Create database tables" />
-        <echo message="load_registries --> Load metadata &amp; file format registries into the " />
-        <echo message="                    database" />
         <echo message="" />
         <echo message="clean_backups   --> Remove .bak directories under install directory" />
-        <echo message="clean_database  --> Remove DSpace database tables, destroying data" />
         <echo message="test_database   --> Attempt to connect to the DSpace database in order to verify that configuration is correct" />
         <echo message="" />
         <echo message="" />
@@ -182,10 +177,10 @@ Common usage:
     </target>
 
     <!-- ============================================================= -->
-    <!-- Update an installation (except database)                      -->
+    <!-- Update an installation                                        -->
     <!-- ============================================================= -->
 
-    <target name="update" depends="update_configs,update_code,update_webapps,update_registries" description="Update installed code and web applications (without clobbering data/config)">
+    <target name="update" depends="update_configs,update_code,update_webapps" description="Update installed code and web applications (without clobbering data/config)">
     </target>
 
     <!-- ============================================================= -->
@@ -536,7 +531,7 @@ Common usage:
 
 
     <!-- ============================================================= -->
-    <!-- Update an installation (except database)                      -->
+    <!-- Update core code only (no webapps or configs)                 -->
     <!-- ============================================================= -->
 
     <target name="update_code" description="Update installed code (without clobbering data/config)">
@@ -800,161 +795,6 @@ Common usage:
         </java>
     </target>
 
-
-    <!-- ============================================================= -->
-    <!-- Create the database tables                                    -->
-    <!-- ============================================================= -->
-
-    <!-- We execute InitializeDatabase, passing in the simple log4j properties
-    -  file in etc/ and the DSpace configuration file using system
-    -  properties -->
-    <target name="setup_database" description="Create database tables">
-
-        <!-- Load the Schema -->
-        <java classname="org.dspace.app.launcher.ScriptLauncher" classpathref="class.path" fork="yes" failonerror="yes">
-            <sysproperty key="log4j.configuration" value="file:config/log4j-console.properties" />
-            <sysproperty key="dspace.log.init.disable" value="true" />
-            <sysproperty key="dspace.configuration" value="${config}" />
-            <arg value="setup-database" />
-            <arg value="database_schema.sql" />
-        </java>
-
-        <!-- Add the browse tables -->
-        <java classname="org.dspace.browse.InitializeBrowseDatabase" classpathref="class.path" fork="yes" failonerror="yes">
-            <sysproperty key="log4j.configuration" value="file:config/log4j-console.properties" />
-            <sysproperty key="dspace.log.init.disable" value="true" />
-            <sysproperty key="dspace.configuration" value="${config}" />
-            <arg value="database_schema.sql" />
-        </java>
-
-    </target>
-
-
-    <!-- ============================================================= -->
-    <!-- Remove the database tables                                    -->
-    <!-- ============================================================= -->
-
-    <!-- We execute InitializeDatabase, passing in the simple log4j properties
-    -  file in etc/ and the DSpace configuration file using system
-    -  properties -->
-    <target name="clean_database" description="Removes DSpace database tables, destroying data">
-
-        <java classname="org.dspace.browse.InitializeBrowseDatabase" classpathref="class.path" fork="yes" failonerror="yes">
-            <sysproperty key="log4j.configuration" value="file:config/log4j-console.properties" />
-            <sysproperty key="dspace.log.init.disable" value="true" />
-            <sysproperty key="dspace.configuration" value="${config}" />
-            <arg value="clean-database.sql" />
-        </java>
-
-        <java classname="org.dspace.app.launcher.ScriptLauncher" classpathref="class.path" fork="yes" failonerror="yes">
-            <sysproperty key="log4j.configuration" value="file:config/log4j-console.properties" />
-            <sysproperty key="dspace.log.init.disable" value="true" />
-            <sysproperty key="dspace.configuration" value="${config}" />
-            <arg value="clean-database" />
-            <arg value="clean-database.sql" />
-        </java>
-
-    </target>
-
-
-    <!-- ============================================================= -->
-    <!-- Load the initial contents of the registries into the database -->
-    <!-- ============================================================= -->
-
-    <!-- Loads bitstream format and Dublin Core type registries -->
-    <target name="load_registries" description="Load initial contents of registries">
-
-        <!-- first import the bitstream registry -->
-        <java classname="org.dspace.administer.RegistryLoader" classpathref="class.path" fork="yes" failonerror="yes">
-            <sysproperty key="log4j.configuration" value="file:config/log4j-console.properties" />
-            <sysproperty key="dspace.log.init.disable" value="true" />
-            <sysproperty key="dspace.configuration" value="${config}" />
-            <arg value="-bitstream" />
-            <arg value="${dspace.dir}/config/registries/bitstream-formats.xml" />
-        </java>
-
-        <!-- finally import the metadata elements -->
-        <java classname="org.dspace.administer.MetadataImporter" classpathref="class.path" fork="yes" failonerror="yes">
-            <sysproperty key="log4j.configuration" value="file:config/log4j-console.properties" />
-            <sysproperty key="dspace.log.init.disable" value="true" />
-            <sysproperty key="dspace.configuration" value="${config}" />
-            <arg line="-f '${dspace.dir}/config/registries/dublin-core-types.xml'" />
-        </java>
-
-        <!-- Import the new DCTerms schema -->
-        <java classname="org.dspace.administer.MetadataImporter" classpathref="class.path" fork="yes" failonerror="yes">
-            <sysproperty key="log4j.configuration" value="file:config/log4j-console.properties" />
-            <sysproperty key="dspace.log.init.disable" value="true" />
-            <sysproperty key="dspace.configuration" value="${config}" />
-            <arg line="-f '${dspace.dir}/config/registries/dcterms-types.xml'" />
-        </java>
-
-        <!-- Import the new EPerson schema -->
-        <java classname="org.dspace.administer.MetadataImporter" classpathref="class.path" fork="yes" failonerror="yes">
-            <sysproperty key="log4j.configuration" value="file:config/log4j-console.properties" />
-            <sysproperty key="dspace.log.init.disable" value="true" />
-            <sysproperty key="dspace.configuration" value="${config}" />
-            <arg line="-f '${dspace.dir}/config/registries/eperson-types.xml'" />
-        </java>
-
-        <!-- FIXME: this should be more modular -->
-        <!-- import the SWORD required metadata -->
-        <java classname="org.dspace.administer.MetadataImporter" classpathref="class.path" fork="yes" failonerror="yes">
-            <sysproperty key="log4j.configuration" value="file:config/log4j-console.properties" />
-            <sysproperty key="dspace.log.init.disable" value="true" />
-            <sysproperty key="dspace.configuration" value="${config}" />
-            <arg line="-f '${dspace.dir}/config/registries/sword-metadata.xml'" />
-        </java>
-
-    </target>
-
-
-    <!-- ============================================================= -->
-    <!-- Update contents of the registries -->
-    <!-- ============================================================= -->
-
-    <target name="update_registries" description="Update the metadata registries">
-        <java classname="org.dspace.administer.MetadataImporter" classpathref="class.path" fork="yes" failonerror="yes">
-            <sysproperty key="log4j.configuration" value="file:config/log4j-console.properties" />
-            <sysproperty key="dspace.log.init.disable" value="true" />
-            <sysproperty key="dspace.configuration" value="${config}" />
-            <arg line="-f '${dspace.dir}/config/registries/dublin-core-types.xml'" />
-            <arg line="-u"/>
-        </java>
-
-        <java classname="org.dspace.administer.MetadataImporter" classpathref="class.path" fork="yes" failonerror="yes">
-            <sysproperty key="log4j.configuration" value="file:config/log4j-console.properties" />
-            <sysproperty key="dspace.log.init.disable" value="true" />
-            <sysproperty key="dspace.configuration" value="${config}" />
-            <arg line="-f '${dspace.dir}/config/registries/dcterms-types.xml'" />
-            <arg line="-u"/>
-        </java>
-
-        <java classname="org.dspace.administer.MetadataImporter" classpathref="class.path" fork="yes" failonerror="yes">
-            <sysproperty key="log4j.configuration" value="file:config/log4j-console.properties" />
-            <sysproperty key="dspace.log.init.disable" value="true" />
-            <sysproperty key="dspace.configuration" value="${config}" />
-            <arg line="-f '${dspace.dir}/config/registries/eperson-types.xml'" />
-            <arg line="-u"/>
-        </java>
-
-        <java classname="org.dspace.administer.MetadataImporter" classpathref="class.path" fork="yes" failonerror="yes">
-            <sysproperty key="log4j.configuration" value="file:config/log4j-console.properties" />
-            <sysproperty key="dspace.log.init.disable" value="true" />
-            <sysproperty key="dspace.configuration" value="${config}" />
-            <arg line="-f '${dspace.dir}/config/registries/sword-metadata.xml'" />
-            <arg line="-u"/>
-        </java>
-
-        <java classname="org.dspace.administer.MetadataImporter" classpathref="class.path" fork="yes" failonerror="yes">
-            <sysproperty key="log4j.configuration" value="file:config/log4j-console.properties" />
-            <sysproperty key="dspace.log.init.disable" value="true" />
-            <sysproperty key="dspace.configuration" value="${config}" />
-            <arg line="-f '${dspace.dir}/config/registries/workflow-types.xml'" />
-            <arg line="-u"/>
-        </java>
-    </target>
-
     <!-- ============================================================= -->
     <!-- Install fresh code but do not touch the database              -->
     <!-- ============================================================= -->
@@ -1011,7 +851,7 @@ Common usage:
     <!-- ============================================================= -->
 
     <target name="fresh_install"
-            depends="init_installation,init_configs,test_database,setup_database,load_registries,install_code"
+            depends="init_installation,init_configs,test_database,install_code"
             description="Do a fresh install of the system, overwriting any data">
 
         <delete failonerror="no">
@@ -1030,22 +870,9 @@ Common usage:
             <arg line="dsrun org.dspace.eperson.Group"/>
         </java>
 
-        <java classname="org.dspace.browse.IndexBrowse" classpathref="class.path" fork="yes" failonerror="yes">
-            <sysproperty key="log4j.configuration" value="file:config/log4j-console.properties" />
-            <sysproperty key="dspace.log.init.disable" value="true" />
-            <sysproperty key="dspace.configuration" value="${config}" />
-            <arg line="-f" />
-        </java>
-
-        <java classname="org.dspace.search.DSIndexer" classpathref="class.path" fork="yes" failonerror="yes">
-            <sysproperty key="log4j.configuration" value="file:config/log4j-console.properties" />
-            <sysproperty key="dspace.log.init.disable" value="true" />
-            <sysproperty key="dspace.configuration" value="${config}" />
-        </java>
-
         <echo>
 ====================================================================
- The DSpace code has been installed, and the database initialized.
+ The DSpace code has been installed.
 
  To complete installation, you should do the following:
 
@@ -1056,12 +883,13 @@ Common usage:
    the appropriate place for your servlet container.
    (e.g. '$CATALINA_HOME/webapps' for Tomcat)
 
+ * Start up your servlet container (e.g. Tomcat). DSpace now will
+   initialize the database on the first startup.
+
  * Make an initial administrator account (an e-person) in DSpace:
 
    ${dspace.dir}/bin/dspace create-administrator
 
- * Start up your servlet container (Tomcat etc.)
-
  You should then be able to access your DSpace's 'home page':
 
    ${dspace.url}
@@ -1071,7 +899,7 @@ Common usage:
 
     </target>
 
-	<!-- installes GeoCity resolution database -->
+	<!-- installs GeoCity resolution database -->
     <target name="update_geolite">
         <echo>Downloading: ${geolite}</echo>
         <trycatch property="geolite.error">