hazza/DSpace
1
0
Fork 0
mirror of https://github.com/DSpace/DSpace.git synced 2025-10-07 01:54:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #724 from tdonohue/DS-2243

Browse Source 
DS-2243: Rework/Refactor XMLWorkflow migration so that it works properly with Flyway
This commit is contained in:
Tim Donohue
2014-11-04 14:00:39 -06:00
parent c45aca9e97 c4bb99a2a7
commit b4291c274f
9 changed files with 213 additions and 339 deletions
Download Patch File Download Diff File Expand all files Collapse all files

178
dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseManager.java
View File

@@ -899,184 +899,6 @@ public class DatabaseManager
return (table == null) ? null : table.toLowerCase();
}
////////////////////////////////////////
// SQL loading methods
////////////////////////////////////////
/**
* Load SQL into the RDBMS.
*
* @param sql
* The SQL to load.
* throws SQLException
* If a database error occurs
*/
public static void loadSql(String sql) throws SQLException
{
try
{
loadSql(new StringReader(sql));
}
catch (IOException ioe)
{
log.error("IOE loadSQL Error - ",ioe);
}
}
/**
* Load SQL from a reader into the RDBMS.
*
* @param r
* The Reader from which to read the SQL.
* @throws SQLException
* If a database error occurs
* @throws IOException
* If an error occurs obtaining data from the reader
*/
public static void loadSql(Reader r) throws SQLException, IOException
{
BufferedReader reader = new BufferedReader(r);
StringBuilder sqlBuilder = new StringBuilder();
String sql = null;
String line = null;
Connection connection = null;
Statement statement = null;
try
{
connection = getConnection();
connection.setAutoCommit(true);
statement = connection.createStatement();
boolean inquote = false;
while ((line = reader.readLine()) != null)
{
// Look for comments
int commentStart = line.indexOf("--");
String input = (commentStart != -1) ? line.substring(0, commentStart) : line;
// Empty line, skip
if (input.trim().equals(""))
{
continue;
}
// Put it on the SQL buffer
sqlBuilder.append(input.replace(';', ' ')); // remove all semicolons
// from sql file!
// Add a space
sqlBuilder.append(" ");
// More to come?
// Look for quotes
int index = 0;
int count = 0;
int inputlen = input.length();
while ((index = input.indexOf('\'', count)) != -1)
{
// Flip the value of inquote
inquote = !inquote;
// Move the index
count = index + 1;
// Make sure we do not exceed the string length
if (count >= inputlen)
{
break;
}
}
// If we are in a quote, keep going
// Note that this is STILL a simple heuristic that is not
// guaranteed to be correct
if (inquote)
{
continue;
}
int endMarker = input.indexOf(';', index);
if (endMarker == -1)
{
continue;
}
sql = sqlBuilder.toString();
if (log.isDebugEnabled())
{
log.debug("Running database query \"" + sql + "\"");
}
try
{
// Use execute, not executeQuery (which expects results) or
// executeUpdate
statement.execute(sql);
}
catch (SQLWarning sqlw)
{
if (log.isDebugEnabled())
{
log.debug("Got SQL Warning: " + sqlw, sqlw);
}
}
catch (SQLException sqle)
{
String msg = "Got SQL Exception: " + sqle;
String sqlmessage = sqle.getMessage();
// These are Postgres-isms:
// There's no easy way to check if a table exists before
// creating it, so we always drop tables, then create them
boolean isDrop = ((sql != null) && (sqlmessage != null)
&& (sql.toUpperCase().startsWith("DROP"))
&& (sqlmessage.indexOf("does not exist") != -1));
// Creating a view causes a bogus warning
boolean isNoResults = ((sql != null)
&& (sqlmessage != null)
&& (sql.toUpperCase().startsWith("CREATE VIEW")
|| sql.toUpperCase().startsWith("CREATE FUNCTION"))
&& (sqlmessage.indexOf("No results were returned") != -1));
// If the messages are bogus, give them a low priority
if (isDrop || isNoResults)
{
log.debug(msg, sqle);
}
// Otherwise, we need to know!
else
{
log.warn(msg, sqle);
}
}
// Reset SQL buffer
sqlBuilder = new StringBuilder();
sql = null;
}
}
finally
{
if (connection != null)
{
connection.close();
}
if (statement != null)
{
statement.close();
}
}
}
////////////////////////////////////////
// Helper methods
////////////////////////////////////////

128
dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseUtils.java
View File

@@ -16,6 +16,7 @@ import java.sql.DatabaseMetaData;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import javax.sql.DataSource;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
@@ -26,6 +27,9 @@ import org.dspace.discovery.SearchServiceException;
import org.flywaydb.core.Flyway;
import org.flywaydb.core.api.FlywayException;
import org.flywaydb.core.api.MigrationInfo;
import org.flywaydb.core.internal.dbsupport.DbSupport;
import org.flywaydb.core.internal.dbsupport.DbSupportFactory;
import org.flywaydb.core.internal.dbsupport.SqlScript;
import org.flywaydb.core.internal.info.MigrationInfoDumper;
/**
@@ -66,8 +70,8 @@ public class DatabaseUtils
if (argv.length != 1)
{
System.out.println("\nDatabase action argument is missing.");
System.out.println("Valid actions include: 'test', 'info', 'migrate', 'repair' or 'clean'");
System.out.println("Or, type 'database help' for more information.");
System.out.println("Valid actions include: 'test', 'info', 'migrate', 'migrate-ignored, 'repair' or 'clean'");
System.out.println("Or, type 'database help' for more information.\n");
System.exit(1);
}
@@ -153,6 +157,18 @@ public class DatabaseUtils
DatabaseManager.getDbName();
System.out.println("Done.");
}
// "migrate-ignored" = Manually run any "ignored" Database migrations (if any)
else if(argv[0].equalsIgnoreCase("migrate-ignored"))
{
System.out.println("\nDatabase URL: " + url);
System.out.println("Migrating database to latest version AND running previously \"Ignored\" migrations... (Check logs for details)");
Connection connection = dataSource.getConnection();
// Update the database, but set "outOfOrder=true"
updateDatabase(dataSource, connection, true);
connection.close();
System.out.println("Done.");
}
// "repair" = Run Flyway repair script
else if(argv[0].equalsIgnoreCase("repair"))
{
@@ -183,12 +199,14 @@ public class DatabaseUtils
else
{
System.out.println("\nUsage: database [action]");
System.out.println("Valid actions include: 'test', 'info', 'migrate', 'repair' or 'clean'");
System.out.println(" - test = Test database connection is OK");
System.out.println(" - info = Describe basic info about Database (type, version, driver)");
System.out.println(" - migrate = Migrate the Database to the latest version");
System.out.println(" - repair = Attempt to repair any previously failed database migrations (see also Flyway repair command)");
System.out.println(" - clean = Destroy all data (Warning there is no going back!)");
System.out.println("Valid actions include: 'test', 'info', 'migrate', 'migrate-ignored, 'repair' or 'clean'");
System.out.println(" - test = Test database connection is OK");
System.out.println(" - info = Describe basic info about database (type, version, driver, migrations run)");
System.out.println(" - migrate = Migrate the Database to the latest version");
System.out.println(" - migrate-ignored = If any migrations are \"Ignored\", run them AND migrate to the latest version");
System.out.println(" - repair = Attempt to repair any previously failed database migrations");
System.out.println(" - clean = Destroy all data and tables in Database (WARNING there is no going back!)");
System.out.println("");
}
System.exit(0);
@@ -229,17 +247,28 @@ public class DatabaseUtils
String scriptFolder = DatabaseManager.findDbKeyword(meta);
connection.close();
// Set location where Flyway will load DB scripts from (based on DB Type)
// e.g. [dspace.dir]/etc/[dbtype]/
String scriptPath = ConfigurationManager.getProperty("dspace.dir") +
System.getProperty("file.separator") + "etc" +
System.getProperty("file.separator") + "migrations" +
System.getProperty("file.separator") + scriptFolder;
// Determine location(s) where Flyway will load all DB migrations
ArrayList<String> scriptLocations = new ArrayList<String>();
// Flyway will look in "scriptPath" for SQL migrations AND
// in 'org.dspace.storage.rdbms.migration.*' for Java migrations
log.info("Loading Flyway DB migrations from " + scriptPath + " and Package 'org.dspace.storage.rdbms.migration.*'");
flywaydb.setLocations("filesystem:" + scriptPath, "classpath:org.dspace.storage.rdbms.migration");
// First, add location of SQL migrations (based on DB Type)
// e.g. [dspace.dir]/etc/[dbtype]/
scriptLocations.add("filesystem:" + ConfigurationManager.getProperty("dspace.dir") +
"/etc/migrations/" + scriptFolder);
// Next, add the Java package where Flyway will load Java migrations from
scriptLocations.add("classpath:org.dspace.storage.rdbms.migration");
// Special scenario: If XMLWorkflows are enabled, we need to run its migration(s)
// as it REQUIRES database schema changes. XMLWorkflow uses Java migrations
// which first check whether the XMLWorkflow tables already exist
if (ConfigurationManager.getProperty("workflow", "workflow.framework").equals("xmlworkflow"))
{
scriptLocations.add("classpath:org.dspace.storage.rdbms.xmlworkflow");
}
// Now tell Flyway which locations to load SQL / Java migrations from
log.info("Loading Flyway DB migrations from: " + StringUtils.join(scriptLocations, ", "));
flywaydb.setLocations(scriptLocations.toArray(new String[scriptLocations.size()]));
// Set flyway callbacks (i.e. classes which are called post-DB migration and similar)
// In this situation, we have a Registry Updater that runs PRE-migration
@@ -273,12 +302,42 @@ public class DatabaseUtils
*/
protected static synchronized void updateDatabase(DataSource datasource, Connection connection)
throws SQLException
{
// By default, never run migrations out of order
updateDatabase(datasource, connection, false);
}
/**
* Ensures the current database is up-to-date with regards
* to the latest DSpace DB schema. If the scheme is not up-to-date,
* then any necessary database migrations are performed.
* <P>
* FlywayDB (http://flywaydb.org/) is used to perform database migrations.
* If a Flyway DB migration fails it will be rolled back to the last
* successful migration, and any errors will be logged.
*
* @param datasource
* DataSource object (retrieved from DatabaseManager())
* @param connection
* Database connection
* @param outOfOrder
* If true, Flyway will run any lower version migrations that were previously "ignored".
* If false, Flyway will only run new migrations with a higher version number.
* @throws SQLException
* If database cannot be upgraded.
*/
protected static synchronized void updateDatabase(DataSource datasource, Connection connection, boolean outOfOrder)
throws SQLException
{
try
{
// Setup Flyway API against our database
Flyway flyway = setupFlyway(datasource);
// Set whethe Flyway will run migrations "out of order". By default, this is false,
// and Flyway ONLY runs migrations that have a higher version number.
flyway.setOutOfOrder(outOfOrder);
// Does the necessary Flyway table ("schema_version") exist in this database?
// If not, then this is the first time Flyway has run, and we need to initialize
// NOTE: search is case sensitive, as flyway table name is ALWAYS lowercase,
@@ -327,7 +386,7 @@ public class DatabaseUtils
catch(FlywayException fe)
{
// If any FlywayException (Runtime) is thrown, change it to a SQLException
throw new SQLException("Flyway initilization/migration error occurred", fe);
throw new SQLException("Flyway migration error occurred", fe);
}
}
@@ -762,6 +821,37 @@ public class DatabaseUtils
return exists;
}
/**
* Execute a block of SQL against the current database connection.
* <P>
* The SQL is executed using the Flyway SQL parser.
*
* @param connection
* Current Database Connection
* @param sqlToExecute
* The actual SQL to execute as a String
* @throws SQLException
* If a database error occurs
*/
public static void executeSql(Connection connection, String sqlToExecute) throws SQLException
{
try
{
// Create a Flyway DbSupport object (based on our connection)
// This is how Flyway determines the database *type* (e.g. Postgres vs Oracle)
DbSupport dbSupport = DbSupportFactory.createDbSupport(connection, false);
// Load our SQL string & execute via Flyway's SQL parser
SqlScript script = new SqlScript(sqlToExecute, dbSupport);
script.execute(dbSupport.getJdbcTemplate());
}
catch(FlywayException fe)
{
// If any FlywayException (Runtime) is thrown, change it to a SQLException
throw new SQLException("Flyway executeSql() error occurred", fe);
}
}
/**
* Whether or not to tell Discovery to reindex itself based on the updated
* database.

112
dspace-api/src/main/java/org/dspace/storage/rdbms/migration/V5_0_2014_01_01__XMLWorkflow_Migration.java
View File

@@ -1,112 +0,0 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.storage.rdbms.migration;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
import java.sql.SQLException;
import org.apache.commons.lang.StringUtils;
import org.dspace.core.ConfigurationManager;
import org.dspace.storage.rdbms.DatabaseManager;
import org.dspace.storage.rdbms.DatabaseUtils;
import org.flywaydb.core.api.migration.MigrationChecksumProvider;
import org.flywaydb.core.api.migration.jdbc.JdbcMigration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This class automatically migrates your DSpace Database to use the
* XML-based Configurable Workflow system whenever it is enabled.
* (i.e. workflow.framework=xmlworkflow in workflow.cfg)
* <P>
* This class represents a Flyway DB Java Migration
* http://flywaydb.org/documentation/migration/java.html
* <P>
* It can upgrade a 4.0 (or above) version of DSpace to use the XMLWorkflow.
*
* @author Tim Donohue
*/
public class V5_0_2014_01_01__XMLWorkflow_Migration
implements JdbcMigration, MigrationChecksumProvider
{
/** logging category */
private static final Logger log = LoggerFactory.getLogger(V5_0_2014_01_01__XMLWorkflow_Migration.class);
// Size of migration script run
Integer migration_file_size = -1;
/**
* Actually migrate the existing database
* @param connection
*/
@Override
public void migrate(Connection connection)
throws IOException, SQLException
{
// Get the name of the Schema that the DSpace Database is using
String schema = ConfigurationManager.getProperty("db.schema");
if(StringUtils.isBlank(schema)){
schema = null;
}
// Check if XML Workflow is enabled in workflow.cfg
if (ConfigurationManager.getProperty("workflow", "workflow.framework").equals("xmlworkflow"))
{
//First, ensure both Migration Scripts exist (in src/main/resources)
String dbMigrationScript = V5_0_2014_01_01__XMLWorkflow_Migration.class.getPackage().getName() +
".xmlworkflow." + DatabaseManager.getDbKeyword() + "xml_workflow_migration.sql";
String dataMigrationScript = V5_0_2014_01_01__XMLWorkflow_Migration.class.getPackage().getName() +
".xmlworkflow." + DatabaseManager.getDbKeyword() + "data_workflow_migration.sql";
File dbMigration = new File(dbMigrationScript);
File dataMigration = new File(dataMigrationScript);
if(!dbMigration.exists() || !dataMigration.exists())
{
throw new IOException("Cannot locate XMLWorkflow Database Migration scripts in 'src/main/resources'. " +
"UNABLE TO ENABLE XML WORKFLOW IN DATABASE.");
}
// Now, check if the XMLWorkflow table (cwf_workflowitem) already exists in this database
// If XMLWorkflow Table does NOT exist in this database, then lets do the migration!
if (!DatabaseUtils.tableExists(connection, "cwf_workflowitem"))
{
// Run the DB Migration first!
DatabaseManager.loadSql(new FileReader(dbMigration));
// Then migrate any existing data (i.e. workflows)
DatabaseManager.loadSql(new FileReader(dataMigration));
// Assuming both succeeded, save the size of the dbMigration script for getChecksum() below
migration_file_size = (int) dbMigration.length();
}
}//end if XML Workflow enabled
}
/**
* Return the checksum to be associated with this Migration
* in the Flyway database table (schema_version).
* @return checksum as an Integer
*/
@Override
public Integer getChecksum()
{
if (ConfigurationManager.getProperty("workflow", "workflow.framework").equals("xmlworkflow"))
{
return migration_file_size;
}
else
{
// This migration script wasn't actually run. Just return -1
return -1;
}
}
}

104
dspace-api/src/main/java/org/dspace/storage/rdbms/xmlworkflow/V5_0_2014_11_04__Enable_XMLWorkflow_Migration.java Normal file
View File

@@ -0,0 +1,104 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.storage.rdbms.xmlworkflow;
import java.io.IOException;
import java.sql.Connection;
import java.sql.SQLException;
import org.dspace.core.ConfigurationManager;
import org.dspace.core.Constants;
import org.dspace.storage.rdbms.DatabaseManager;
import org.dspace.storage.rdbms.DatabaseUtils;
import org.flywaydb.core.api.migration.MigrationChecksumProvider;
import org.flywaydb.core.api.migration.jdbc.JdbcMigration;
import org.flywaydb.core.internal.util.scanner.classpath.ClassPathResource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This class automatically migrates your DSpace Database to use the
* XML-based Configurable Workflow system whenever it is enabled.
* (i.e. workflow.framework=xmlworkflow in workflow.cfg)
* <P>
* Because XML-based Configurable Workflow existed prior to our migration, this
* class first checks for the existence of the "cwf_workflowitem" table before
* running any migrations.
* <P>
* This class represents a Flyway DB Java Migration
* http://flywaydb.org/documentation/migration/java.html
* <P>
* It can upgrade a 5.0 (or above) version of DSpace to use the XMLWorkflow.
*
* @author Tim Donohue
*/
public class V5_0_2014_11_04__Enable_XMLWorkflow_Migration
implements JdbcMigration, MigrationChecksumProvider
{
/** logging category */
private static final Logger log = LoggerFactory.getLogger(V5_0_2014_11_04__Enable_XMLWorkflow_Migration.class);
// Size of migration script run
Integer migration_file_size = -1;
/**
* Actually migrate the existing database
* @param connection
*/
@Override
public void migrate(Connection connection)
throws IOException, SQLException
{
// Make sure XML Workflow is enabled in workflow.cfg before proceeding
if (ConfigurationManager.getProperty("workflow", "workflow.framework").equals("xmlworkflow"))
{
// Now, check if the XMLWorkflow table (cwf_workflowitem) already exists in this database
// If XMLWorkflow Table does NOT exist in this database, then lets do the migration!
// If XMLWorkflow Table ALREADY exists, then this migration is a noop, we assume you manually ran the sql scripts
if (!DatabaseUtils.tableExists(connection, "cwf_workflowitem"))
{
// Determine path of this migration class (as the SQL scripts
// we will run are based on this path under /src/main/resources)
String packagePath = V5_0_2014_11_04__Enable_XMLWorkflow_Migration.class.getPackage().getName().replace(".", "/");
// Get the contents of our DB Schema migration script, based on path & DB type
// (e.g. /src/main/resources/[path-to-this-class]/postgres/xml_workflow_migration.sql)
String dbMigrateSQL = new ClassPathResource(packagePath + "/" +
DatabaseManager.getDbKeyword() +
"/xml_workflow_migration.sql", getClass().getClassLoader()).loadAsString(Constants.DEFAULT_ENCODING);
// Actually execute the Database schema migration SQL
// This will create the necessary tables for the XMLWorkflow feature
DatabaseUtils.executeSql(connection, dbMigrateSQL);
// Get the contents of our data migration script, based on path & DB type
// (e.g. /src/main/resources/[path-to-this-class]/postgres/data_workflow_migration.sql)
String dataMigrateSQL = new ClassPathResource(packagePath + "/" +
DatabaseManager.getDbKeyword() +
"/data_workflow_migration.sql", getClass().getClassLoader()).loadAsString(Constants.DEFAULT_ENCODING);
// Actually execute the Data migration SQL
// This will migrate all existing traditional workflows to the new XMLWorkflow system & tables
DatabaseUtils.executeSql(connection, dataMigrateSQL);
// Assuming both succeeded, save the size of the scripts for getChecksum() below
migration_file_size = dbMigrateSQL.length() + dataMigrateSQL.length();
}
}
}
/**
* Return the checksum to be associated with this Migration
* in the Flyway database table (schema_version).
* @return checksum as an Integer
*/
@Override
public Integer getChecksum()
{
return migration_file_size;
}
}

0
dspace-api/src/main/resources/org/dspace/storage/rdbms/migration/xmlworkflow/oracle/data_workflow_migration.sql → dspace-api/src/main/resources/org/dspace/storage/rdbms/xmlworkflow/oracle/data_workflow_migration.sql
View File

0
dspace-api/src/main/resources/org/dspace/storage/rdbms/migration/xmlworkflow/oracle/xml_workflow_migration.sql → dspace-api/src/main/resources/org/dspace/storage/rdbms/xmlworkflow/oracle/xml_workflow_migration.sql
View File

0
dspace-api/src/main/resources/org/dspace/storage/rdbms/migration/xmlworkflow/postgres/data_workflow_migration.sql → dspace-api/src/main/resources/org/dspace/storage/rdbms/xmlworkflow/postgres/data_workflow_migration.sql
View File

0
dspace-api/src/main/resources/org/dspace/storage/rdbms/migration/xmlworkflow/postgres/xml_workflow_migration.sql → dspace-api/src/main/resources/org/dspace/storage/rdbms/xmlworkflow/postgres/xml_workflow_migration.sql
View File

30
dspace-api/src/test/java/org/dspace/storage/rdbms/DatabaseManagerTest.java
View File

@@ -506,36 +506,6 @@ public class DatabaseManagerTest
}
*/
/**
* Test of loadSql method, of class DatabaseManager.
*/
/*
@Test
public void testLoadSql_String() throws Exception
{
System.out.println("loadSql");
String sql = "";
DatabaseManager.loadSql(sql);
// TODO review the generated test code and remove the default call to fail.
fail("The test case is a prototype.");
}
*/
/**
* Test of loadSql method, of class DatabaseManager.
*/
/*
@Test
public void testLoadSql_Reader() throws Exception
{
System.out.println("loadSql");
Reader r = null;
DatabaseManager.loadSql(r);
// TODO review the generated test code and remove the default call to fail.
fail("The test case is a prototype.");
}
*/
/**
* Test of process method, of class DatabaseManager.
*/