From aa582848a49a31f20d91adc79e1688783f01f9ec Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Tue, 19 Jul 2016 10:07:26 -0500 Subject: [PATCH 1/8] Add ability to "validate" a database by running Flyway validate directly. Makes for easier testing/validation --- .../dspace/storage/rdbms/DatabaseUtils.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseUtils.java b/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseUtils.java index 4c279b1765..7522fc1b07 100644 --- a/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseUtils.java +++ b/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseUtils.java @@ -86,7 +86,7 @@ public class DatabaseUtils if (argv.length < 1) { System.out.println("\nDatabase action argument is missing."); - System.out.println("Valid actions: 'test', 'info', 'migrate', 'repair' or 'clean'"); + System.out.println("Valid actions: 'test', 'info', 'migrate', 'repair', 'validate' or 'clean'"); System.out.println("\nOr, type 'database help' for more information.\n"); System.exit(1); } @@ -285,6 +285,23 @@ public class DatabaseUtils System.exit(1); } } + // "validate" = Run Flyway validation to check for database errors/issues + else if(argv[0].equalsIgnoreCase("validate")) + { + try (Connection connection = dataSource.getConnection();) + { + System.out.println("\nDatabase URL: " + connection.getMetaData().getURL()); + System.out.println("Attempting to validate database status (and migration checksums) via FlywayDB... (Check dspace logs for more details)"); + flyway.validate(); + System.out.println("Done."); + } + catch(SQLException|FlywayException e) + { + System.err.println("Validation exception:"); + e.printStackTrace(); + System.exit(1); + } + } // "clean" = Run Flyway clean script else if(argv[0].equalsIgnoreCase("clean")) { @@ -348,6 +365,7 @@ public class DatabaseUtils System.out.println(" - info / status = Describe basic info/status about database, including validating the compatibility of this database"); System.out.println(" - migrate = Migrate the database to the latest version"); System.out.println(" - repair = Attempt to repair any previously failed database migrations or checksum mismatches (via Flyway repair)"); + System.out.println(" - validate = Validate current database's migration status (via Flyway validate), validating all migration checksums."); System.out.println(" - clean = DESTROY all data and tables in database (WARNING there is no going back!)"); System.out.println(""); } From 5a61d621008e426d0f9f3b4fe7ca40bd42ee4757 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Tue, 19 Jul 2016 10:08:02 -0500 Subject: [PATCH 2/8] DS-3234: Upgrade to Flyway 4.0.3. Fix minor upgrade issues with FlywayCallback classes. --- dspace-api/pom.xml | 2 +- .../dspace/storage/rdbms/DatabaseRegistryUpdater.java | 10 ---------- .../dspace/storage/rdbms/GroupServiceInitializer.java | 10 ---------- .../dspace/storage/rdbms/PostgreSQLCryptoChecker.java | 10 ---------- .../dspace/storage/rdbms/SiteServiceInitializer.java | 10 ---------- 5 files changed, 1 insertion(+), 41 deletions(-) diff --git a/dspace-api/pom.xml b/dspace-api/pom.xml index 1ce9be2d2e..d96d9ef612 100644 --- a/dspace-api/pom.xml +++ b/dspace-api/pom.xml @@ -635,7 +635,7 @@ org.flywaydb flyway-core - 3.2.1 + 4.0.3 diff --git a/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseRegistryUpdater.java b/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseRegistryUpdater.java index 39e069de89..6e983d5829 100644 --- a/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseRegistryUpdater.java +++ b/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseRegistryUpdater.java @@ -140,16 +140,6 @@ public class DatabaseRegistryUpdater implements FlywayCallback } - @Override - public void beforeInit(Connection connection) { - - } - - @Override - public void afterInit(Connection connection) { - - } - @Override public void beforeBaseline(Connection connection) { diff --git a/dspace-api/src/main/java/org/dspace/storage/rdbms/GroupServiceInitializer.java b/dspace-api/src/main/java/org/dspace/storage/rdbms/GroupServiceInitializer.java index 21e63c2585..c6179bfc53 100644 --- a/dspace-api/src/main/java/org/dspace/storage/rdbms/GroupServiceInitializer.java +++ b/dspace-api/src/main/java/org/dspace/storage/rdbms/GroupServiceInitializer.java @@ -97,16 +97,6 @@ public class GroupServiceInitializer implements FlywayCallback { } - @Override - public void beforeInit(Connection connection) { - - } - - @Override - public void afterInit(Connection connection) { - - } - @Override public void beforeBaseline(Connection connection) { diff --git a/dspace-api/src/main/java/org/dspace/storage/rdbms/PostgreSQLCryptoChecker.java b/dspace-api/src/main/java/org/dspace/storage/rdbms/PostgreSQLCryptoChecker.java index 456cc83c73..3b50c89ed9 100644 --- a/dspace-api/src/main/java/org/dspace/storage/rdbms/PostgreSQLCryptoChecker.java +++ b/dspace-api/src/main/java/org/dspace/storage/rdbms/PostgreSQLCryptoChecker.java @@ -146,16 +146,6 @@ public class PostgreSQLCryptoChecker implements FlywayCallback } - @Override - public void beforeInit(Connection connection) { - - } - - @Override - public void afterInit(Connection connection) { - - } - @Override public void beforeBaseline(Connection connection) { // Before initializing database, check for pgcrypto diff --git a/dspace-api/src/main/java/org/dspace/storage/rdbms/SiteServiceInitializer.java b/dspace-api/src/main/java/org/dspace/storage/rdbms/SiteServiceInitializer.java index 4c37e38c3f..243cc50d53 100644 --- a/dspace-api/src/main/java/org/dspace/storage/rdbms/SiteServiceInitializer.java +++ b/dspace-api/src/main/java/org/dspace/storage/rdbms/SiteServiceInitializer.java @@ -100,16 +100,6 @@ public class SiteServiceInitializer implements FlywayCallback { } - @Override - public void beforeInit(Connection connection) { - - } - - @Override - public void afterInit(Connection connection) { - - } - @Override public void beforeBaseline(Connection connection) { From fe25840b6429e95f52d45c4eca8247eb331ccb66 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Tue, 19 Jul 2016 12:05:51 -0500 Subject: [PATCH 3/8] DS-3234 : Enable new Flyway 4 option to disable 'clean' by default. Refactor commandline tools so 'test' also reports DB issues. --- .../dspace/storage/rdbms/DatabaseUtils.java | 198 +++++++++++------- dspace/config/dspace.cfg | 7 + 2 files changed, 126 insertions(+), 79 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseUtils.java b/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseUtils.java index 7522fc1b07..99f480f31e 100644 --- a/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseUtils.java +++ b/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseUtils.java @@ -80,8 +80,6 @@ public class DatabaseUtils */ public static void main(String[] argv) { - ConfigurationService config = DSpaceServicesFactory.getInstance().getConfigurationService(); - // Usage checks if (argv.length < 1) { @@ -106,15 +104,14 @@ public class DatabaseUtils System.out.println("\nAttempting to connect to database"); try(Connection connection = dataSource.getConnection()) { - // Just do a high level test by getting our configured DataSource and attempting to connect to it - DatabaseMetaData meta = connection.getMetaData(); System.out.println("Connected successfully!"); - System.out.println("Database Software: " + meta.getDatabaseProductName() + " version " + meta.getDatabaseProductVersion()); - System.out.println(" - URL: " + meta.getURL()); - System.out.println(" - Driver: " + meta.getDriverName() + " version " + meta.getDriverVersion()); - System.out.println(" - Username: " + meta.getUserName()); - System.out.println(" - Password: [hidden]"); - System.out.println(" - Schema: " + getSchemaName(connection)); + + // Print basic database connection information + printDBInfo(connection); + + // Print any database warnings/errors found (if any) + printDBWarnings(connection); + System.exit(0); } catch (SQLException sqle) { @@ -129,23 +126,8 @@ public class DatabaseUtils { try(Connection connection = dataSource.getConnection()) { - // Get basic Database info - DatabaseMetaData meta = connection.getMetaData(); - String dbType = getDbType(connection); - System.out.println("\nDatabase Type: " + dbType); - System.out.println("Database URL: " + meta.getURL()); - System.out.println("Database Schema: " + getSchemaName(connection)); - System.out.println("Database Software: " + meta.getDatabaseProductName() + " version " + meta.getDatabaseProductVersion()); - System.out.println("Database Driver: " + meta.getDriverName() + " version " + meta.getDriverVersion()); - - // For Postgres, report whether pgcrypto is installed - // (If it isn't, we'll also write out warnings...see below) - if(dbType.equals(DBMS_POSTGRES)) - { - boolean pgcryptoUpToDate = PostgresUtils.isPgcryptoUpToDate(); - Double pgcryptoVersion = PostgresUtils.getPgcryptoInstalledVersion(connection); - System.out.println("PostgreSQL '" + PostgresUtils.PGCRYPTO + "' extension installed/up-to-date? " + pgcryptoUpToDate + " " + ((pgcryptoVersion!=null) ? "(version=" + pgcryptoVersion + ")" : "(not installed)")); - } + // Print basic Database info + printDBInfo(connection); // Get info table from Flyway System.out.println("\n" + MigrationInfoDumper.dumpToAsciiTable(flyway.info().all())); @@ -166,55 +148,9 @@ public class DatabaseUtils } } - // For PostgreSQL databases, we need to check for the 'pgcrypto' extension. - // If it is NOT properly installed, we'll need to warn the user, as DSpace will be unable to proceed. - if(dbType.equals(DBMS_POSTGRES)) - { - // Get version of pgcrypto available in this postgres instance - Double pgcryptoAvailable = PostgresUtils.getPgcryptoAvailableVersion(connection); - - // Generic requirements message - String requirementsMsg = "\n** DSpace REQUIRES PostgreSQL >= " + PostgresUtils.POSTGRES_VERSION + " AND " + PostgresUtils.PGCRYPTO + " extension >= " + PostgresUtils.PGCRYPTO_VERSION + " **\n"; - - // Check if installed in PostgreSQL & a supported version - if(pgcryptoAvailable!=null && pgcryptoAvailable.compareTo(PostgresUtils.PGCRYPTO_VERSION)>=0) - { - // We now know it's available in this Postgres. Let's see if it is installed in this database. - Double pgcryptoInstalled = PostgresUtils.getPgcryptoInstalledVersion(connection); - - // Check if installed in database, but outdated version - if(pgcryptoInstalled!=null && pgcryptoInstalled.compareTo(PostgresUtils.PGCRYPTO_VERSION)<0) - { - System.out.println("\nWARNING: PostgreSQL '" + PostgresUtils.PGCRYPTO + "' extension is OUTDATED (installed version=" + pgcryptoInstalled + ", available version = " + pgcryptoAvailable + ")."); - System.out.println(requirementsMsg); - System.out.println("To update it, please connect to your DSpace database as a 'superuser' and manually run the following command: "); - System.out.println("\n ALTER EXTENSION " + PostgresUtils.PGCRYPTO + " UPDATE TO '" + pgcryptoAvailable + "';\n"); - } - else if(pgcryptoInstalled==null) // If it's not installed in database - { - System.out.println("\nWARNING: PostgreSQL '" + PostgresUtils.PGCRYPTO + "' extension is NOT INSTALLED on this database."); - System.out.println(requirementsMsg); - System.out.println("To install it, please connect to your DSpace database as a 'superuser' and manually run the following command: "); - System.out.println("\n CREATE EXTENSION " + PostgresUtils.PGCRYPTO + ";\n"); - } - } - // Check if installed in Postgres, but an unsupported version - else if(pgcryptoAvailable!=null && pgcryptoAvailable.compareTo(PostgresUtils.PGCRYPTO_VERSION)<0) - { - System.out.println("\nWARNING: UNSUPPORTED version of PostgreSQL '" + PostgresUtils.PGCRYPTO + "' extension found (version=" + pgcryptoAvailable + ")."); - System.out.println(requirementsMsg); - System.out.println("Make sure you are running a supported version of PostgreSQL, and then install " + PostgresUtils.PGCRYPTO + " version >= " + PostgresUtils.PGCRYPTO_VERSION); - System.out.println("The '" + PostgresUtils.PGCRYPTO + "' extension is often provided in the 'postgresql-contrib' package for your operating system."); - } - else if(pgcryptoAvailable==null) // If it's not installed in Postgres - { - System.out.println("\nWARNING: PostgreSQL '" + PostgresUtils.PGCRYPTO + "' extension is NOT AVAILABLE. Please install it into this PostgreSQL instance."); - System.out.println(requirementsMsg); - System.out.println("The '" + PostgresUtils.PGCRYPTO + "' extension is often provided in the 'postgresql-contrib' package for your operating system."); - System.out.println("Once the extension is installed globally, please connect to your DSpace database as a 'superuser' and manually run the following command: "); - System.out.println("\n CREATE EXTENSION " + PostgresUtils.PGCRYPTO + ";\n"); - } - } + // Print any database warnings/errors found (if any) + printDBWarnings(connection); + System.exit(0); } catch (SQLException e) { @@ -260,6 +196,7 @@ public class DatabaseUtils updateDatabase(dataSource, connection); } System.out.println("Done."); + System.exit(0); } catch(SQLException e) { @@ -277,6 +214,7 @@ public class DatabaseUtils System.out.println("Attempting to repair any previously failed migrations (or mismatched checksums) via FlywayDB... (Check dspace logs for details)"); flyway.repair(); System.out.println("Done."); + System.exit(0); } catch(SQLException|FlywayException e) { @@ -291,9 +229,10 @@ public class DatabaseUtils try (Connection connection = dataSource.getConnection();) { System.out.println("\nDatabase URL: " + connection.getMetaData().getURL()); - System.out.println("Attempting to validate database status (and migration checksums) via FlywayDB... (Check dspace logs for more details)"); + System.out.println("Attempting to validate database status (and migration checksums) via FlywayDB..."); flyway.validate(); - System.out.println("Done."); + System.out.println("No errors thrown. Validation succeeded. (Check dspace logs for more details)"); + System.exit(0); } catch(SQLException|FlywayException e) { @@ -305,6 +244,14 @@ public class DatabaseUtils // "clean" = Run Flyway clean script else if(argv[0].equalsIgnoreCase("clean")) { + // If clean is disabled, return immediately + if(flyway.isCleanDisabled()) + { + System.out.println("WARNING: 'clean' command is currently disabled, as it is dangerous to run in Production scenarios!"); + System.out.println("\n In order to run a 'clean' you first must enable it in your DSpace config by specifying 'db.cleanDisabled=false'."); + System.exit(1); + } + try (Connection connection = dataSource.getConnection()) { String dbType = getDbType(connection); @@ -348,6 +295,7 @@ public class DatabaseUtils System.out.println("Scrubbing database clean... (Check dspace logs for details)"); cleanDatabase(flyway, dataSource); System.out.println("Done."); + System.exit(0); } } catch(SQLException e) @@ -366,8 +314,9 @@ public class DatabaseUtils System.out.println(" - migrate = Migrate the database to the latest version"); System.out.println(" - repair = Attempt to repair any previously failed database migrations or checksum mismatches (via Flyway repair)"); System.out.println(" - validate = Validate current database's migration status (via Flyway validate), validating all migration checksums."); - System.out.println(" - clean = DESTROY all data and tables in database (WARNING there is no going back!)"); + System.out.println(" - clean = DESTROY all data and tables in database (WARNING there is no going back!). Requires 'db.cleanDisabled=false' setting in config."); System.out.println(""); + System.exit(0); } } @@ -379,7 +328,95 @@ public class DatabaseUtils } } + /** + * Print basic information about the current database to System.out. + * This is utilized by both the 'test' and 'info' commandline options. + * @param connection current database connection + * @throws SQLException if database error occurs + */ + private static void printDBInfo(Connection connection) throws SQLException + { + // Get basic Database info from connection + DatabaseMetaData meta = connection.getMetaData(); + String dbType = getDbType(connection); + System.out.println("\nDatabase Type: " + dbType); + System.out.println("Database URL: " + meta.getURL()); + System.out.println("Database Schema: " + getSchemaName(connection)); + System.out.println("Database Username: " + meta.getUserName()); + System.out.println("Database Software: " + meta.getDatabaseProductName() + " version " + meta.getDatabaseProductVersion()); + System.out.println("Database Driver: " + meta.getDriverName() + " version " + meta.getDriverVersion()); + // For Postgres, report whether pgcrypto is installed + // (If it isn't, we'll also write out warnings...see below) + if(dbType.equals(DBMS_POSTGRES)) + { + boolean pgcryptoUpToDate = PostgresUtils.isPgcryptoUpToDate(); + Double pgcryptoVersion = PostgresUtils.getPgcryptoInstalledVersion(connection); + System.out.println("PostgreSQL '" + PostgresUtils.PGCRYPTO + "' extension installed/up-to-date? " + pgcryptoUpToDate + " " + ((pgcryptoVersion!=null) ? "(version=" + pgcryptoVersion + ")" : "(not installed)")); + } + } + + /** + * Print any warnings about current database setup to System.err (if any). + * This is utilized by both the 'test' and 'info' commandline options. + * @param connection current database connection + * @throws SQLException if database error occurs + */ + private static void printDBWarnings(Connection connection) throws SQLException + { + // Get the DB Type + String dbType = getDbType(connection); + + // For PostgreSQL databases, we need to check for the 'pgcrypto' extension. + // If it is NOT properly installed, we'll need to warn the user, as DSpace will be unable to proceed. + if(dbType.equals(DBMS_POSTGRES)) + { + // Get version of pgcrypto available in this postgres instance + Double pgcryptoAvailable = PostgresUtils.getPgcryptoAvailableVersion(connection); + + // Generic requirements message + String requirementsMsg = "\n** DSpace REQUIRES PostgreSQL >= " + PostgresUtils.POSTGRES_VERSION + " AND " + PostgresUtils.PGCRYPTO + " extension >= " + PostgresUtils.PGCRYPTO_VERSION + " **\n"; + + // Check if installed in PostgreSQL & a supported version + if(pgcryptoAvailable!=null && pgcryptoAvailable.compareTo(PostgresUtils.PGCRYPTO_VERSION)>=0) + { + // We now know it's available in this Postgres. Let's see if it is installed in this database. + Double pgcryptoInstalled = PostgresUtils.getPgcryptoInstalledVersion(connection); + + // Check if installed in database, but outdated version + if(pgcryptoInstalled!=null && pgcryptoInstalled.compareTo(PostgresUtils.PGCRYPTO_VERSION)<0) + { + System.err.println("\nWARNING: Required PostgreSQL '" + PostgresUtils.PGCRYPTO + "' extension is OUTDATED (installed version=" + pgcryptoInstalled + ", available version = " + pgcryptoAvailable + ")."); + System.err.println(requirementsMsg); + System.err.println("To update it, please connect to your DSpace database as a 'superuser' and manually run the following command: "); + System.err.println("\n ALTER EXTENSION " + PostgresUtils.PGCRYPTO + " UPDATE TO '" + pgcryptoAvailable + "';\n"); + } + else if(pgcryptoInstalled==null) // If it's not installed in database + { + System.err.println("\nWARNING: Required PostgreSQL '" + PostgresUtils.PGCRYPTO + "' extension is NOT INSTALLED on this database."); + System.err.println(requirementsMsg); + System.err.println("To install it, please connect to your DSpace database as a 'superuser' and manually run the following command: "); + System.err.println("\n CREATE EXTENSION " + PostgresUtils.PGCRYPTO + ";\n"); + } + } + // Check if installed in Postgres, but an unsupported version + else if(pgcryptoAvailable!=null && pgcryptoAvailable.compareTo(PostgresUtils.PGCRYPTO_VERSION)<0) + { + System.err.println("\nWARNING: UNSUPPORTED version of PostgreSQL '" + PostgresUtils.PGCRYPTO + "' extension found (version=" + pgcryptoAvailable + ")."); + System.err.println(requirementsMsg); + System.err.println("Make sure you are running a supported version of PostgreSQL, and then install " + PostgresUtils.PGCRYPTO + " version >= " + PostgresUtils.PGCRYPTO_VERSION); + System.err.println("The '" + PostgresUtils.PGCRYPTO + "' extension is often provided in the 'postgresql-contrib' package for your operating system."); + } + else if(pgcryptoAvailable==null) // If it's not installed in Postgres + { + System.err.println("\nWARNING: PostgreSQL '" + PostgresUtils.PGCRYPTO + "' extension is NOT AVAILABLE. Please install it into this PostgreSQL instance."); + System.err.println(requirementsMsg); + System.err.println("The '" + PostgresUtils.PGCRYPTO + "' extension is often provided in the 'postgresql-contrib' package for your operating system."); + System.err.println("Once the extension is installed globally, please connect to your DSpace database as a 'superuser' and manually run the following command: "); + System.err.println("\n CREATE EXTENSION " + PostgresUtils.PGCRYPTO + ";\n"); + } + } + } /** * Setup/Initialize the Flyway API to run against our DSpace database @@ -402,6 +439,9 @@ public class DatabaseUtils flywaydb.setDataSource(datasource); flywaydb.setEncoding("UTF-8"); + // Default cleanDisabled to "true" (which disallows the ability to run 'database clean') + flywaydb.setCleanDisabled(config.getBooleanProperty("db.cleanDisabled", true)); + // Migration scripts are based on DBMS Keyword (see full path below) String dbType = getDbType(connection); connection.close(); diff --git a/dspace/config/dspace.cfg b/dspace/config/dspace.cfg index cbb6130870..a72f947d46 100644 --- a/dspace/config/dspace.cfg +++ b/dspace/config/dspace.cfg @@ -102,6 +102,13 @@ db.maxidle = -1 # pool. #db.jndi = jdbc/dspace +# Whether or not to allow for an entire 'clean' of the DSpace database. +# Enabling it would allow your database owner to destroy all DSpace data, tables, etc +# by simply running 'dspace database clean' from commandline. +# WARNING: NEVER ENABLE IN PRODUCTION. This tool is for development/testing only. +# (Defaults to 'true', i.e. disabled) +# db.cleanDisabled = true + ##### Email settings ###### # SMTP mail server (allows DSpace to send email notifications) From 24e7a5a5ec5f23063ac39dbd8a6a0d8b6e6ecaf8 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Tue, 19 Jul 2016 12:07:34 -0500 Subject: [PATCH 4/8] DS-3234: No longer upgrade database from Ant. However the Ant 'database test' will now report DB issues PRIOR to upgrading your DB (see previous commit) --- dspace/src/main/config/build.xml | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/dspace/src/main/config/build.xml b/dspace/src/main/config/build.xml index e2c8ffb8cd..b0e0e048ca 100644 --- a/dspace/src/main/config/build.xml +++ b/dspace/src/main/config/build.xml @@ -135,7 +135,6 @@ Common usage: - @@ -176,7 +175,7 @@ Common usage: - + @@ -797,16 +796,6 @@ Common usage: - - - - - - - - - - @@ -864,7 +853,7 @@ Common usage: From 5de418ad03fd9f22b079fd9d063ae47eaca556ea Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Tue, 19 Jul 2016 12:43:42 -0500 Subject: [PATCH 5/8] DS-3234 : Ensure test_database is called for updates. Change Ant log4j-console settings to WARN or above. --- .../dspace/storage/rdbms/DatabaseUtils.java | 34 +++++++++---------- dspace/config/log4j-console.properties | 8 +++-- dspace/src/main/config/build.xml | 3 +- 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseUtils.java b/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseUtils.java index 99f480f31e..854d297a73 100644 --- a/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseUtils.java +++ b/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseUtils.java @@ -386,34 +386,34 @@ public class DatabaseUtils // Check if installed in database, but outdated version if(pgcryptoInstalled!=null && pgcryptoInstalled.compareTo(PostgresUtils.PGCRYPTO_VERSION)<0) { - System.err.println("\nWARNING: Required PostgreSQL '" + PostgresUtils.PGCRYPTO + "' extension is OUTDATED (installed version=" + pgcryptoInstalled + ", available version = " + pgcryptoAvailable + ")."); - System.err.println(requirementsMsg); - System.err.println("To update it, please connect to your DSpace database as a 'superuser' and manually run the following command: "); - System.err.println("\n ALTER EXTENSION " + PostgresUtils.PGCRYPTO + " UPDATE TO '" + pgcryptoAvailable + "';\n"); + System.out.println("\nWARNING: Required PostgreSQL '" + PostgresUtils.PGCRYPTO + "' extension is OUTDATED (installed version=" + pgcryptoInstalled + ", available version = " + pgcryptoAvailable + ")."); + System.out.println(requirementsMsg); + System.out.println("To update it, please connect to your DSpace database as a 'superuser' and manually run the following command: "); + System.out.println("\n ALTER EXTENSION " + PostgresUtils.PGCRYPTO + " UPDATE TO '" + pgcryptoAvailable + "';\n"); } else if(pgcryptoInstalled==null) // If it's not installed in database { - System.err.println("\nWARNING: Required PostgreSQL '" + PostgresUtils.PGCRYPTO + "' extension is NOT INSTALLED on this database."); - System.err.println(requirementsMsg); - System.err.println("To install it, please connect to your DSpace database as a 'superuser' and manually run the following command: "); - System.err.println("\n CREATE EXTENSION " + PostgresUtils.PGCRYPTO + ";\n"); + System.out.println("\nWARNING: Required PostgreSQL '" + PostgresUtils.PGCRYPTO + "' extension is NOT INSTALLED on this database."); + System.out.println(requirementsMsg); + System.out.println("To install it, please connect to your DSpace database as a 'superuser' and manually run the following command: "); + System.out.println("\n CREATE EXTENSION " + PostgresUtils.PGCRYPTO + ";\n"); } } // Check if installed in Postgres, but an unsupported version else if(pgcryptoAvailable!=null && pgcryptoAvailable.compareTo(PostgresUtils.PGCRYPTO_VERSION)<0) { - System.err.println("\nWARNING: UNSUPPORTED version of PostgreSQL '" + PostgresUtils.PGCRYPTO + "' extension found (version=" + pgcryptoAvailable + ")."); - System.err.println(requirementsMsg); - System.err.println("Make sure you are running a supported version of PostgreSQL, and then install " + PostgresUtils.PGCRYPTO + " version >= " + PostgresUtils.PGCRYPTO_VERSION); - System.err.println("The '" + PostgresUtils.PGCRYPTO + "' extension is often provided in the 'postgresql-contrib' package for your operating system."); + System.out.println("\nWARNING: UNSUPPORTED version of PostgreSQL '" + PostgresUtils.PGCRYPTO + "' extension found (version=" + pgcryptoAvailable + ")."); + System.out.println(requirementsMsg); + System.out.println("Make sure you are running a supported version of PostgreSQL, and then install " + PostgresUtils.PGCRYPTO + " version >= " + PostgresUtils.PGCRYPTO_VERSION); + System.out.println("The '" + PostgresUtils.PGCRYPTO + "' extension is often provided in the 'postgresql-contrib' package for your operating system."); } else if(pgcryptoAvailable==null) // If it's not installed in Postgres { - System.err.println("\nWARNING: PostgreSQL '" + PostgresUtils.PGCRYPTO + "' extension is NOT AVAILABLE. Please install it into this PostgreSQL instance."); - System.err.println(requirementsMsg); - System.err.println("The '" + PostgresUtils.PGCRYPTO + "' extension is often provided in the 'postgresql-contrib' package for your operating system."); - System.err.println("Once the extension is installed globally, please connect to your DSpace database as a 'superuser' and manually run the following command: "); - System.err.println("\n CREATE EXTENSION " + PostgresUtils.PGCRYPTO + ";\n"); + System.out.println("\nWARNING: PostgreSQL '" + PostgresUtils.PGCRYPTO + "' extension is NOT AVAILABLE. Please install it into this PostgreSQL instance."); + System.out.println(requirementsMsg); + System.out.println("The '" + PostgresUtils.PGCRYPTO + "' extension is often provided in the 'postgresql-contrib' package for your operating system."); + System.out.println("Once the extension is installed globally, please connect to your DSpace database as a 'superuser' and manually run the following command: "); + System.out.println("\n CREATE EXTENSION " + PostgresUtils.PGCRYPTO + ";\n"); } } } diff --git a/dspace/config/log4j-console.properties b/dspace/config/log4j-console.properties index 6d441dd447..5a52df7c5b 100644 --- a/dspace/config/log4j-console.properties +++ b/dspace/config/log4j-console.properties @@ -6,8 +6,10 @@ # Its goal is to simply output logs to the commandline / console. ############################################################# -# Set root category priority to INFO and its only appender to A1. -log4j.rootCategory=INFO, A1 +# Set root category priority to WARN and its only appender to A1. +# For commandline / ant scripts, we are only concerned about significant warnings/errors +# For the full detail, change this to INFO and re-run Ant. +log4j.rootCategory=WARN, A1 # A1 is set to be a ConsoleAppender. log4j.appender.A1=org.apache.log4j.ConsoleAppender @@ -18,4 +20,4 @@ log4j.appender.A1.layout.ConversionPattern=%d %-5p %c @ %m%n # block passwords from being exposed in Axis logs. # (DEBUG exposes passwords in Basic Auth) -log4j.logger.org.apache.axis.handlers.http.HTTPAuthHandler=INFO \ No newline at end of file +log4j.logger.org.apache.axis.handlers.http.HTTPAuthHandler=INFO diff --git a/dspace/src/main/config/build.xml b/dspace/src/main/config/build.xml index b0e0e048ca..c5a85aa3bd 100644 --- a/dspace/src/main/config/build.xml +++ b/dspace/src/main/config/build.xml @@ -135,6 +135,7 @@ Common usage: + @@ -175,7 +176,7 @@ Common usage: - + From 7d63df81dc5ae558cfdb892f7258c0b44c744ee1 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Tue, 19 Jul 2016 12:57:54 -0500 Subject: [PATCH 6/8] Return proper error status if issues found with database --- .../dspace/storage/rdbms/DatabaseUtils.java | 28 +++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseUtils.java b/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseUtils.java index 854d297a73..210a6ce2a5 100644 --- a/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseUtils.java +++ b/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseUtils.java @@ -110,8 +110,13 @@ public class DatabaseUtils printDBInfo(connection); // Print any database warnings/errors found (if any) - printDBWarnings(connection); - System.exit(0); + boolean issueFound = printDBIssues(connection); + + // If issues found, exit with an error status (even if connection succeeded). + if(issueFound) + System.exit(1); + else + System.exit(0); } catch (SQLException sqle) { @@ -149,8 +154,13 @@ public class DatabaseUtils } // Print any database warnings/errors found (if any) - printDBWarnings(connection); - System.exit(0); + boolean issueFound = printDBIssues(connection); + + // If issues found, exit with an error status + if(issueFound) + System.exit(1); + else + System.exit(0); } catch (SQLException e) { @@ -360,10 +370,13 @@ public class DatabaseUtils * Print any warnings about current database setup to System.err (if any). * This is utilized by both the 'test' and 'info' commandline options. * @param connection current database connection + * @return boolean true if database issues found, false otherwise * @throws SQLException if database error occurs */ - private static void printDBWarnings(Connection connection) throws SQLException + private static boolean printDBIssues(Connection connection) throws SQLException { + boolean issueFound = false; + // Get the DB Type String dbType = getDbType(connection); @@ -390,6 +403,7 @@ public class DatabaseUtils System.out.println(requirementsMsg); System.out.println("To update it, please connect to your DSpace database as a 'superuser' and manually run the following command: "); System.out.println("\n ALTER EXTENSION " + PostgresUtils.PGCRYPTO + " UPDATE TO '" + pgcryptoAvailable + "';\n"); + issueFound = true; } else if(pgcryptoInstalled==null) // If it's not installed in database { @@ -397,6 +411,7 @@ public class DatabaseUtils System.out.println(requirementsMsg); System.out.println("To install it, please connect to your DSpace database as a 'superuser' and manually run the following command: "); System.out.println("\n CREATE EXTENSION " + PostgresUtils.PGCRYPTO + ";\n"); + issueFound = true; } } // Check if installed in Postgres, but an unsupported version @@ -406,6 +421,7 @@ public class DatabaseUtils System.out.println(requirementsMsg); System.out.println("Make sure you are running a supported version of PostgreSQL, and then install " + PostgresUtils.PGCRYPTO + " version >= " + PostgresUtils.PGCRYPTO_VERSION); System.out.println("The '" + PostgresUtils.PGCRYPTO + "' extension is often provided in the 'postgresql-contrib' package for your operating system."); + issueFound = true; } else if(pgcryptoAvailable==null) // If it's not installed in Postgres { @@ -414,8 +430,10 @@ public class DatabaseUtils System.out.println("The '" + PostgresUtils.PGCRYPTO + "' extension is often provided in the 'postgresql-contrib' package for your operating system."); System.out.println("Once the extension is installed globally, please connect to your DSpace database as a 'superuser' and manually run the following command: "); System.out.println("\n CREATE EXTENSION " + PostgresUtils.PGCRYPTO + ";\n"); + issueFound = true; } } + return issueFound; } /** From d1695521c56141579ee2626ebe82c6123b9dcda9 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Tue, 19 Jul 2016 16:36:40 -0500 Subject: [PATCH 7/8] DS-3234: Initialize database when Context first loads. Ensures starting Tomcat initializes database again. --- .../main/java/org/dspace/core/Context.java | 19 ++++++++++++++++--- .../dspace/storage/rdbms/DatabaseUtils.java | 6 +++--- dspace/src/main/config/build.xml | 2 +- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/core/Context.java b/dspace-api/src/main/java/org/dspace/core/Context.java index 56512ce986..5f87d20998 100644 --- a/dspace-api/src/main/java/org/dspace/core/Context.java +++ b/dspace-api/src/main/java/org/dspace/core/Context.java @@ -16,6 +16,7 @@ import org.dspace.event.Event; import org.dspace.event.factory.EventServiceFactory; import org.dspace.event.service.EventService; import org.dspace.storage.rdbms.DatabaseConfigVO; +import org.dspace.storage.rdbms.DatabaseUtils; import org.dspace.utils.DSpace; import org.springframework.util.CollectionUtils; @@ -83,6 +84,21 @@ public class Context private DBConnection dbConnection; + static + { + // Before initializing a Context object, we need to ensure the database + // is up-to-date. This ensures any outstanding Flyway migrations are run + // PRIOR to Hibernate initializing (occurs when DBConnection is loaded in init() below). + try + { + DatabaseUtils.updateDatabase(); + } + catch(SQLException sqle) + { + log.fatal("Cannot initialize database via Flyway!", sqle); + } + } + protected Context(EventService eventService, DBConnection dbConnection) { this.eventService = eventService; this.dbConnection = dbConnection; @@ -93,9 +109,6 @@ public class Context /** * Construct a new context object with default options. A database connection is opened. * No user is authenticated. - * - * @exception SQLException - * if there was an error obtaining a database connection */ public Context() { diff --git a/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseUtils.java b/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseUtils.java index 210a6ce2a5..3811c8bd28 100644 --- a/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseUtils.java +++ b/dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseUtils.java @@ -257,8 +257,8 @@ public class DatabaseUtils // If clean is disabled, return immediately if(flyway.isCleanDisabled()) { - System.out.println("WARNING: 'clean' command is currently disabled, as it is dangerous to run in Production scenarios!"); - System.out.println("\n In order to run a 'clean' you first must enable it in your DSpace config by specifying 'db.cleanDisabled=false'."); + System.out.println("\nWARNING: 'clean' command is currently disabled, as it is dangerous to run in Production scenarios!"); + System.out.println("\nIn order to run a 'clean' you first must enable it in your DSpace config by specifying 'db.cleanDisabled=false'.\n"); System.exit(1); } @@ -278,7 +278,7 @@ public class DatabaseUtils System.out.println("\nERROR: The database user '" + username + "' does not have sufficient privileges to run a 'database clean' (via Flyway)."); System.out.println("\nIn order to run a 'clean', the database user MUST have 'superuser' privileges"); System.out.println("OR the '" + PostgresUtils.PGCRYPTO + "' extension must be installed in a separate schema (see documentation)."); - System.out.println("\nOptionally, you could also manually remove the '" + PostgresUtils.PGCRYPTO + "' extension first (DROP EXTENSION " + PostgresUtils.PGCRYPTO + " CASCADE), then rerun the 'clean'"); + System.out.println("\nOptionally, you could also manually remove the '" + PostgresUtils.PGCRYPTO + "' extension first (DROP EXTENSION " + PostgresUtils.PGCRYPTO + " CASCADE;), then rerun the 'clean'"); System.exit(1); } } diff --git a/dspace/src/main/config/build.xml b/dspace/src/main/config/build.xml index c5a85aa3bd..e587121c69 100644 --- a/dspace/src/main/config/build.xml +++ b/dspace/src/main/config/build.xml @@ -176,7 +176,7 @@ Common usage: - + From 2dfaa4bf574502178c0b808cd7de3db1528dc16e Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Wed, 3 Aug 2016 13:39:35 -0500 Subject: [PATCH 8/8] Enhance/clarify comments around 'db.cleanDisabled' setting --- dspace/config/dspace.cfg | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dspace/config/dspace.cfg b/dspace/config/dspace.cfg index a72f947d46..9195813c68 100644 --- a/dspace/config/dspace.cfg +++ b/dspace/config/dspace.cfg @@ -103,10 +103,11 @@ db.maxidle = -1 #db.jndi = jdbc/dspace # Whether or not to allow for an entire 'clean' of the DSpace database. -# Enabling it would allow your database owner to destroy all DSpace data, tables, etc -# by simply running 'dspace database clean' from commandline. -# WARNING: NEVER ENABLE IN PRODUCTION. This tool is for development/testing only. -# (Defaults to 'true', i.e. disabled) +# By default, this setting is 'true', which ensures that the 'dspace database clean' command +# does nothing (except return an error message saying clean is disabled) +# Setting this config to 'false' allows your database owner to destroy all DSpace data, tables, etc +# by running 'dspace database clean' from commandline. This is only useful for development/testing. +# WARNING: NEVER SET TO 'false' IN PRODUCTION. # db.cleanDisabled = true ##### Email settings ######