diff --git a/dspace-api/pom.xml b/dspace-api/pom.xml index 1ce9be2d2e..d96d9ef612 100644 --- a/dspace-api/pom.xml +++ b/dspace-api/pom.xml @@ -635,7 +635,7 @@ org.flywaydb flyway-core - 3.2.1 + 4.0.3 diff --git a/dspace-api/src/main/java/org/dspace/core/Context.java b/dspace-api/src/main/java/org/dspace/core/Context.java index 56512ce986..5f87d20998 100644 --- a/dspace-api/src/main/java/org/dspace/core/Context.java +++ b/dspace-api/src/main/java/org/dspace/core/Context.java @@ -16,6 +16,7 @@ import org.dspace.event.Event; import org.dspace.event.factory.EventServiceFactory; import org.dspace.event.service.EventService; import org.dspace.storage.rdbms.DatabaseConfigVO; +import org.dspace.storage.rdbms.DatabaseUtils; import org.dspace.utils.DSpace; import org.springframework.util.CollectionUtils; @@ -83,6 +84,21 @@ public class Context private DBConnection dbConnection; + static + { + // Before initializing a Context object, we need to ensure the database + // is up-to-date. This ensures any outstanding Flyway migrations are run + // PRIOR to Hibernate initializing (occurs when DBConnection is loaded in init() below). + try + { + DatabaseUtils.updateDatabase(); + } + catch(SQLException sqle) + { + log.fatal("Cannot initialize database via Flyway!", sqle); + } + } + protected Context(EventService eventService, DBConnection dbConnection) { this.eventService = eventService; this.dbConnection = dbConnection; @@ -93,9 +109,6 @@ public class Context /** * Construct a new context object with default options. A database connection is opened. * No user is authenticated. - * - * @exception SQLException - * if there was an error obtaining a database connection */ public Context() { diff --git a/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseRegistryUpdater.java b/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseRegistryUpdater.java index 39e069de89..6e983d5829 100644 --- a/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseRegistryUpdater.java +++ b/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseRegistryUpdater.java @@ -140,16 +140,6 @@ public class DatabaseRegistryUpdater implements FlywayCallback } - @Override - public void beforeInit(Connection connection) { - - } - - @Override - public void afterInit(Connection connection) { - - } - @Override public void beforeBaseline(Connection connection) { diff --git a/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseUtils.java b/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseUtils.java index 4c279b1765..3811c8bd28 100644 --- a/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseUtils.java +++ b/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseUtils.java @@ -80,13 +80,11 @@ public class DatabaseUtils */ public static void main(String[] argv) { - ConfigurationService config = DSpaceServicesFactory.getInstance().getConfigurationService(); - // Usage checks if (argv.length < 1) { System.out.println("\nDatabase action argument is missing."); - System.out.println("Valid actions: 'test', 'info', 'migrate', 'repair' or 'clean'"); + System.out.println("Valid actions: 'test', 'info', 'migrate', 'repair', 'validate' or 'clean'"); System.out.println("\nOr, type 'database help' for more information.\n"); System.exit(1); } @@ -106,15 +104,19 @@ public class DatabaseUtils System.out.println("\nAttempting to connect to database"); try(Connection connection = dataSource.getConnection()) { - // Just do a high level test by getting our configured DataSource and attempting to connect to it - DatabaseMetaData meta = connection.getMetaData(); System.out.println("Connected successfully!"); - System.out.println("Database Software: " + meta.getDatabaseProductName() + " version " + meta.getDatabaseProductVersion()); - System.out.println(" - URL: " + meta.getURL()); - System.out.println(" - Driver: " + meta.getDriverName() + " version " + meta.getDriverVersion()); - System.out.println(" - Username: " + meta.getUserName()); - System.out.println(" - Password: [hidden]"); - System.out.println(" - Schema: " + getSchemaName(connection)); + + // Print basic database connection information + printDBInfo(connection); + + // Print any database warnings/errors found (if any) + boolean issueFound = printDBIssues(connection); + + // If issues found, exit with an error status (even if connection succeeded). + if(issueFound) + System.exit(1); + else + System.exit(0); } catch (SQLException sqle) { @@ -129,23 +131,8 @@ public class DatabaseUtils { try(Connection connection = dataSource.getConnection()) { - // Get basic Database info - DatabaseMetaData meta = connection.getMetaData(); - String dbType = getDbType(connection); - System.out.println("\nDatabase Type: " + dbType); - System.out.println("Database URL: " + meta.getURL()); - System.out.println("Database Schema: " + getSchemaName(connection)); - System.out.println("Database Software: " + meta.getDatabaseProductName() + " version " + meta.getDatabaseProductVersion()); - System.out.println("Database Driver: " + meta.getDriverName() + " version " + meta.getDriverVersion()); - - // For Postgres, report whether pgcrypto is installed - // (If it isn't, we'll also write out warnings...see below) - if(dbType.equals(DBMS_POSTGRES)) - { - boolean pgcryptoUpToDate = PostgresUtils.isPgcryptoUpToDate(); - Double pgcryptoVersion = PostgresUtils.getPgcryptoInstalledVersion(connection); - System.out.println("PostgreSQL '" + PostgresUtils.PGCRYPTO + "' extension installed/up-to-date? " + pgcryptoUpToDate + " " + ((pgcryptoVersion!=null) ? "(version=" + pgcryptoVersion + ")" : "(not installed)")); - } + // Print basic Database info + printDBInfo(connection); // Get info table from Flyway System.out.println("\n" + MigrationInfoDumper.dumpToAsciiTable(flyway.info().all())); @@ -166,55 +153,14 @@ public class DatabaseUtils } } - // For PostgreSQL databases, we need to check for the 'pgcrypto' extension. - // If it is NOT properly installed, we'll need to warn the user, as DSpace will be unable to proceed. - if(dbType.equals(DBMS_POSTGRES)) - { - // Get version of pgcrypto available in this postgres instance - Double pgcryptoAvailable = PostgresUtils.getPgcryptoAvailableVersion(connection); + // Print any database warnings/errors found (if any) + boolean issueFound = printDBIssues(connection); - // Generic requirements message - String requirementsMsg = "\n** DSpace REQUIRES PostgreSQL >= " + PostgresUtils.POSTGRES_VERSION + " AND " + PostgresUtils.PGCRYPTO + " extension >= " + PostgresUtils.PGCRYPTO_VERSION + " **\n"; - - // Check if installed in PostgreSQL & a supported version - if(pgcryptoAvailable!=null && pgcryptoAvailable.compareTo(PostgresUtils.PGCRYPTO_VERSION)>=0) - { - // We now know it's available in this Postgres. Let's see if it is installed in this database. - Double pgcryptoInstalled = PostgresUtils.getPgcryptoInstalledVersion(connection); - - // Check if installed in database, but outdated version - if(pgcryptoInstalled!=null && pgcryptoInstalled.compareTo(PostgresUtils.PGCRYPTO_VERSION)<0) - { - System.out.println("\nWARNING: PostgreSQL '" + PostgresUtils.PGCRYPTO + "' extension is OUTDATED (installed version=" + pgcryptoInstalled + ", available version = " + pgcryptoAvailable + ")."); - System.out.println(requirementsMsg); - System.out.println("To update it, please connect to your DSpace database as a 'superuser' and manually run the following command: "); - System.out.println("\n ALTER EXTENSION " + PostgresUtils.PGCRYPTO + " UPDATE TO '" + pgcryptoAvailable + "';\n"); - } - else if(pgcryptoInstalled==null) // If it's not installed in database - { - System.out.println("\nWARNING: PostgreSQL '" + PostgresUtils.PGCRYPTO + "' extension is NOT INSTALLED on this database."); - System.out.println(requirementsMsg); - System.out.println("To install it, please connect to your DSpace database as a 'superuser' and manually run the following command: "); - System.out.println("\n CREATE EXTENSION " + PostgresUtils.PGCRYPTO + ";\n"); - } - } - // Check if installed in Postgres, but an unsupported version - else if(pgcryptoAvailable!=null && pgcryptoAvailable.compareTo(PostgresUtils.PGCRYPTO_VERSION)<0) - { - System.out.println("\nWARNING: UNSUPPORTED version of PostgreSQL '" + PostgresUtils.PGCRYPTO + "' extension found (version=" + pgcryptoAvailable + ")."); - System.out.println(requirementsMsg); - System.out.println("Make sure you are running a supported version of PostgreSQL, and then install " + PostgresUtils.PGCRYPTO + " version >= " + PostgresUtils.PGCRYPTO_VERSION); - System.out.println("The '" + PostgresUtils.PGCRYPTO + "' extension is often provided in the 'postgresql-contrib' package for your operating system."); - } - else if(pgcryptoAvailable==null) // If it's not installed in Postgres - { - System.out.println("\nWARNING: PostgreSQL '" + PostgresUtils.PGCRYPTO + "' extension is NOT AVAILABLE. Please install it into this PostgreSQL instance."); - System.out.println(requirementsMsg); - System.out.println("The '" + PostgresUtils.PGCRYPTO + "' extension is often provided in the 'postgresql-contrib' package for your operating system."); - System.out.println("Once the extension is installed globally, please connect to your DSpace database as a 'superuser' and manually run the following command: "); - System.out.println("\n CREATE EXTENSION " + PostgresUtils.PGCRYPTO + ";\n"); - } - } + // If issues found, exit with an error status + if(issueFound) + System.exit(1); + else + System.exit(0); } catch (SQLException e) { @@ -260,6 +206,7 @@ public class DatabaseUtils updateDatabase(dataSource, connection); } System.out.println("Done."); + System.exit(0); } catch(SQLException e) { @@ -277,6 +224,7 @@ public class DatabaseUtils System.out.println("Attempting to repair any previously failed migrations (or mismatched checksums) via FlywayDB... (Check dspace logs for details)"); flyway.repair(); System.out.println("Done."); + System.exit(0); } catch(SQLException|FlywayException e) { @@ -285,9 +233,35 @@ public class DatabaseUtils System.exit(1); } } + // "validate" = Run Flyway validation to check for database errors/issues + else if(argv[0].equalsIgnoreCase("validate")) + { + try (Connection connection = dataSource.getConnection();) + { + System.out.println("\nDatabase URL: " + connection.getMetaData().getURL()); + System.out.println("Attempting to validate database status (and migration checksums) via FlywayDB..."); + flyway.validate(); + System.out.println("No errors thrown. Validation succeeded. (Check dspace logs for more details)"); + System.exit(0); + } + catch(SQLException|FlywayException e) + { + System.err.println("Validation exception:"); + e.printStackTrace(); + System.exit(1); + } + } // "clean" = Run Flyway clean script else if(argv[0].equalsIgnoreCase("clean")) { + // If clean is disabled, return immediately + if(flyway.isCleanDisabled()) + { + System.out.println("\nWARNING: 'clean' command is currently disabled, as it is dangerous to run in Production scenarios!"); + System.out.println("\nIn order to run a 'clean' you first must enable it in your DSpace config by specifying 'db.cleanDisabled=false'.\n"); + System.exit(1); + } + try (Connection connection = dataSource.getConnection()) { String dbType = getDbType(connection); @@ -304,7 +278,7 @@ public class DatabaseUtils System.out.println("\nERROR: The database user '" + username + "' does not have sufficient privileges to run a 'database clean' (via Flyway)."); System.out.println("\nIn order to run a 'clean', the database user MUST have 'superuser' privileges"); System.out.println("OR the '" + PostgresUtils.PGCRYPTO + "' extension must be installed in a separate schema (see documentation)."); - System.out.println("\nOptionally, you could also manually remove the '" + PostgresUtils.PGCRYPTO + "' extension first (DROP EXTENSION " + PostgresUtils.PGCRYPTO + " CASCADE), then rerun the 'clean'"); + System.out.println("\nOptionally, you could also manually remove the '" + PostgresUtils.PGCRYPTO + "' extension first (DROP EXTENSION " + PostgresUtils.PGCRYPTO + " CASCADE;), then rerun the 'clean'"); System.exit(1); } } @@ -331,6 +305,7 @@ public class DatabaseUtils System.out.println("Scrubbing database clean... (Check dspace logs for details)"); cleanDatabase(flyway, dataSource); System.out.println("Done."); + System.exit(0); } } catch(SQLException e) @@ -348,8 +323,10 @@ public class DatabaseUtils System.out.println(" - info / status = Describe basic info/status about database, including validating the compatibility of this database"); System.out.println(" - migrate = Migrate the database to the latest version"); System.out.println(" - repair = Attempt to repair any previously failed database migrations or checksum mismatches (via Flyway repair)"); - System.out.println(" - clean = DESTROY all data and tables in database (WARNING there is no going back!)"); + System.out.println(" - validate = Validate current database's migration status (via Flyway validate), validating all migration checksums."); + System.out.println(" - clean = DESTROY all data and tables in database (WARNING there is no going back!). Requires 'db.cleanDisabled=false' setting in config."); System.out.println(""); + System.exit(0); } } @@ -361,7 +338,103 @@ public class DatabaseUtils } } + /** + * Print basic information about the current database to System.out. + * This is utilized by both the 'test' and 'info' commandline options. + * @param connection current database connection + * @throws SQLException if database error occurs + */ + private static void printDBInfo(Connection connection) throws SQLException + { + // Get basic Database info from connection + DatabaseMetaData meta = connection.getMetaData(); + String dbType = getDbType(connection); + System.out.println("\nDatabase Type: " + dbType); + System.out.println("Database URL: " + meta.getURL()); + System.out.println("Database Schema: " + getSchemaName(connection)); + System.out.println("Database Username: " + meta.getUserName()); + System.out.println("Database Software: " + meta.getDatabaseProductName() + " version " + meta.getDatabaseProductVersion()); + System.out.println("Database Driver: " + meta.getDriverName() + " version " + meta.getDriverVersion()); + // For Postgres, report whether pgcrypto is installed + // (If it isn't, we'll also write out warnings...see below) + if(dbType.equals(DBMS_POSTGRES)) + { + boolean pgcryptoUpToDate = PostgresUtils.isPgcryptoUpToDate(); + Double pgcryptoVersion = PostgresUtils.getPgcryptoInstalledVersion(connection); + System.out.println("PostgreSQL '" + PostgresUtils.PGCRYPTO + "' extension installed/up-to-date? " + pgcryptoUpToDate + " " + ((pgcryptoVersion!=null) ? "(version=" + pgcryptoVersion + ")" : "(not installed)")); + } + } + + /** + * Print any warnings about current database setup to System.err (if any). + * This is utilized by both the 'test' and 'info' commandline options. + * @param connection current database connection + * @return boolean true if database issues found, false otherwise + * @throws SQLException if database error occurs + */ + private static boolean printDBIssues(Connection connection) throws SQLException + { + boolean issueFound = false; + + // Get the DB Type + String dbType = getDbType(connection); + + // For PostgreSQL databases, we need to check for the 'pgcrypto' extension. + // If it is NOT properly installed, we'll need to warn the user, as DSpace will be unable to proceed. + if(dbType.equals(DBMS_POSTGRES)) + { + // Get version of pgcrypto available in this postgres instance + Double pgcryptoAvailable = PostgresUtils.getPgcryptoAvailableVersion(connection); + + // Generic requirements message + String requirementsMsg = "\n** DSpace REQUIRES PostgreSQL >= " + PostgresUtils.POSTGRES_VERSION + " AND " + PostgresUtils.PGCRYPTO + " extension >= " + PostgresUtils.PGCRYPTO_VERSION + " **\n"; + + // Check if installed in PostgreSQL & a supported version + if(pgcryptoAvailable!=null && pgcryptoAvailable.compareTo(PostgresUtils.PGCRYPTO_VERSION)>=0) + { + // We now know it's available in this Postgres. Let's see if it is installed in this database. + Double pgcryptoInstalled = PostgresUtils.getPgcryptoInstalledVersion(connection); + + // Check if installed in database, but outdated version + if(pgcryptoInstalled!=null && pgcryptoInstalled.compareTo(PostgresUtils.PGCRYPTO_VERSION)<0) + { + System.out.println("\nWARNING: Required PostgreSQL '" + PostgresUtils.PGCRYPTO + "' extension is OUTDATED (installed version=" + pgcryptoInstalled + ", available version = " + pgcryptoAvailable + ")."); + System.out.println(requirementsMsg); + System.out.println("To update it, please connect to your DSpace database as a 'superuser' and manually run the following command: "); + System.out.println("\n ALTER EXTENSION " + PostgresUtils.PGCRYPTO + " UPDATE TO '" + pgcryptoAvailable + "';\n"); + issueFound = true; + } + else if(pgcryptoInstalled==null) // If it's not installed in database + { + System.out.println("\nWARNING: Required PostgreSQL '" + PostgresUtils.PGCRYPTO + "' extension is NOT INSTALLED on this database."); + System.out.println(requirementsMsg); + System.out.println("To install it, please connect to your DSpace database as a 'superuser' and manually run the following command: "); + System.out.println("\n CREATE EXTENSION " + PostgresUtils.PGCRYPTO + ";\n"); + issueFound = true; + } + } + // Check if installed in Postgres, but an unsupported version + else if(pgcryptoAvailable!=null && pgcryptoAvailable.compareTo(PostgresUtils.PGCRYPTO_VERSION)<0) + { + System.out.println("\nWARNING: UNSUPPORTED version of PostgreSQL '" + PostgresUtils.PGCRYPTO + "' extension found (version=" + pgcryptoAvailable + ")."); + System.out.println(requirementsMsg); + System.out.println("Make sure you are running a supported version of PostgreSQL, and then install " + PostgresUtils.PGCRYPTO + " version >= " + PostgresUtils.PGCRYPTO_VERSION); + System.out.println("The '" + PostgresUtils.PGCRYPTO + "' extension is often provided in the 'postgresql-contrib' package for your operating system."); + issueFound = true; + } + else if(pgcryptoAvailable==null) // If it's not installed in Postgres + { + System.out.println("\nWARNING: PostgreSQL '" + PostgresUtils.PGCRYPTO + "' extension is NOT AVAILABLE. Please install it into this PostgreSQL instance."); + System.out.println(requirementsMsg); + System.out.println("The '" + PostgresUtils.PGCRYPTO + "' extension is often provided in the 'postgresql-contrib' package for your operating system."); + System.out.println("Once the extension is installed globally, please connect to your DSpace database as a 'superuser' and manually run the following command: "); + System.out.println("\n CREATE EXTENSION " + PostgresUtils.PGCRYPTO + ";\n"); + issueFound = true; + } + } + return issueFound; + } /** * Setup/Initialize the Flyway API to run against our DSpace database @@ -384,6 +457,9 @@ public class DatabaseUtils flywaydb.setDataSource(datasource); flywaydb.setEncoding("UTF-8"); + // Default cleanDisabled to "true" (which disallows the ability to run 'database clean') + flywaydb.setCleanDisabled(config.getBooleanProperty("db.cleanDisabled", true)); + // Migration scripts are based on DBMS Keyword (see full path below) String dbType = getDbType(connection); connection.close(); diff --git a/dspace-api/src/main/java/org/dspace/storage/rdbms/GroupServiceInitializer.java b/dspace-api/src/main/java/org/dspace/storage/rdbms/GroupServiceInitializer.java index 21e63c2585..c6179bfc53 100644 --- a/dspace-api/src/main/java/org/dspace/storage/rdbms/GroupServiceInitializer.java +++ b/dspace-api/src/main/java/org/dspace/storage/rdbms/GroupServiceInitializer.java @@ -97,16 +97,6 @@ public class GroupServiceInitializer implements FlywayCallback { } - @Override - public void beforeInit(Connection connection) { - - } - - @Override - public void afterInit(Connection connection) { - - } - @Override public void beforeBaseline(Connection connection) { diff --git a/dspace-api/src/main/java/org/dspace/storage/rdbms/PostgreSQLCryptoChecker.java b/dspace-api/src/main/java/org/dspace/storage/rdbms/PostgreSQLCryptoChecker.java index 456cc83c73..3b50c89ed9 100644 --- a/dspace-api/src/main/java/org/dspace/storage/rdbms/PostgreSQLCryptoChecker.java +++ b/dspace-api/src/main/java/org/dspace/storage/rdbms/PostgreSQLCryptoChecker.java @@ -146,16 +146,6 @@ public class PostgreSQLCryptoChecker implements FlywayCallback } - @Override - public void beforeInit(Connection connection) { - - } - - @Override - public void afterInit(Connection connection) { - - } - @Override public void beforeBaseline(Connection connection) { // Before initializing database, check for pgcrypto diff --git a/dspace-api/src/main/java/org/dspace/storage/rdbms/SiteServiceInitializer.java b/dspace-api/src/main/java/org/dspace/storage/rdbms/SiteServiceInitializer.java index 4c37e38c3f..243cc50d53 100644 --- a/dspace-api/src/main/java/org/dspace/storage/rdbms/SiteServiceInitializer.java +++ b/dspace-api/src/main/java/org/dspace/storage/rdbms/SiteServiceInitializer.java @@ -100,16 +100,6 @@ public class SiteServiceInitializer implements FlywayCallback { } - @Override - public void beforeInit(Connection connection) { - - } - - @Override - public void afterInit(Connection connection) { - - } - @Override public void beforeBaseline(Connection connection) { diff --git a/dspace/config/dspace.cfg b/dspace/config/dspace.cfg index cbb6130870..9195813c68 100644 --- a/dspace/config/dspace.cfg +++ b/dspace/config/dspace.cfg @@ -102,6 +102,14 @@ db.maxidle = -1 # pool. #db.jndi = jdbc/dspace +# Whether or not to allow for an entire 'clean' of the DSpace database. +# By default, this setting is 'true', which ensures that the 'dspace database clean' command +# does nothing (except return an error message saying clean is disabled) +# Setting this config to 'false' allows your database owner to destroy all DSpace data, tables, etc +# by running 'dspace database clean' from commandline. This is only useful for development/testing. +# WARNING: NEVER SET TO 'false' IN PRODUCTION. +# db.cleanDisabled = true + ##### Email settings ###### # SMTP mail server (allows DSpace to send email notifications) diff --git a/dspace/config/log4j-console.properties b/dspace/config/log4j-console.properties index 6d441dd447..5a52df7c5b 100644 --- a/dspace/config/log4j-console.properties +++ b/dspace/config/log4j-console.properties @@ -6,8 +6,10 @@ # Its goal is to simply output logs to the commandline / console. ############################################################# -# Set root category priority to INFO and its only appender to A1. -log4j.rootCategory=INFO, A1 +# Set root category priority to WARN and its only appender to A1. +# For commandline / ant scripts, we are only concerned about significant warnings/errors +# For the full detail, change this to INFO and re-run Ant. +log4j.rootCategory=WARN, A1 # A1 is set to be a ConsoleAppender. log4j.appender.A1=org.apache.log4j.ConsoleAppender @@ -18,4 +20,4 @@ log4j.appender.A1.layout.ConversionPattern=%d %-5p %c @ %m%n # block passwords from being exposed in Axis logs. # (DEBUG exposes passwords in Basic Auth) -log4j.logger.org.apache.axis.handlers.http.HTTPAuthHandler=INFO \ No newline at end of file +log4j.logger.org.apache.axis.handlers.http.HTTPAuthHandler=INFO diff --git a/dspace/src/main/config/build.xml b/dspace/src/main/config/build.xml index e2c8ffb8cd..e587121c69 100644 --- a/dspace/src/main/config/build.xml +++ b/dspace/src/main/config/build.xml @@ -135,7 +135,7 @@ Common usage: - + @@ -176,7 +176,7 @@ Common usage: - + @@ -797,16 +797,6 @@ Common usage: - - - - - - - - - - @@ -864,7 +854,7 @@ Common usage: