Merge branch 'master' into DS-3908_Metadata_patch_support

dspace-api/src/main/java/org/dspace/ctask/test/WorkflowReportTest.java

@@ -0,0 +1,39 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE and NOTICE files at the root of the source
+ * tree and available online at
+ *
+ * http://www.dspace.org/license/
+ */
+
+package org.dspace.ctask.test;
+
+import java.io.IOException;
+
+import org.dspace.content.DSpaceObject;
+import org.dspace.curate.AbstractCurationTask;
+import org.dspace.curate.Curator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curation task which simply reports its invocation without changing anything.
+ * Meant for testing.
+ *
+ * @author mhwood
+ */
+public class WorkflowReportTest
+        extends AbstractCurationTask {
+    private static final Logger LOG = LoggerFactory.getLogger(WorkflowReportTest.class);
+
+    @Override
+    public int perform(DSpaceObject dso)
+            throws IOException {
+        LOG.info("Class {} as task {} received 'perform' for object {}",
+                WorkflowReportTest.class.getSimpleName(), taskId, dso);
+        curator.report(String.format(
+                "Class %s as task %s received 'perform' for object %s%n",
+                WorkflowReportTest.class.getSimpleName(), taskId, dso));
+        return Curator.CURATE_SUCCESS;
+    }
+}

dspace-api/src/main/java/org/dspace/curate/CurationCli.java

@@ -9,6 +9,10 @@ package org.dspace.curate;
 
 import java.io.BufferedReader;
 import java.io.FileReader;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintStream;
+import java.io.Writer;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
@@ -18,6 +22,7 @@ import org.apache.commons.cli.CommandLineParser;
 import org.apache.commons.cli.HelpFormatter;
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.PosixParser;
+import org.apache.commons.io.output.NullOutputStream;
 import org.dspace.content.factory.ContentServiceFactory;
 import org.dspace.core.Context;
 import org.dspace.core.factory.CoreServiceFactory;
@@ -57,7 +62,9 @@ public class CurationCli {
         options.addOption("e", "eperson", true,
                           "email address of curating eperson");
         options.addOption("r", "reporter", true,
-                          "reporter to manage results - use '-' to report to console. If absent, no reporting");
+                "relative or absolute path to the desired report file.  "
+                        + "Use '-' to report to console.  "
+                        + "If absent, no reporting");
         options.addOption("s", "scope", true,
                           "transaction scope to impose: use 'object', 'curation', or 'open'. If absent, 'open' " +
                               "applies");
@@ -165,9 +172,17 @@ public class CurationCli {
         }
 
         Curator curator = new Curator();
-        if (reporterName != null) {
+        OutputStream reporter;
-            curator.setReporter(reporterName);
+        if (null == reporterName) {
+            reporter = new NullOutputStream();
+        } else if ("-".equals(reporterName)) {
+            reporter = System.out;
+        } else {
+            reporter = new PrintStream(reporterName);
         }
+        Writer reportWriter = new OutputStreamWriter(reporter);
+        curator.setReporter(reportWriter);
+
         if (scope != null) {
             Curator.TxScope txScope = Curator.TxScope.valueOf(scope.toUpperCase());
             curator.setTransactionScope(txScope);

dspace-api/src/main/java/org/dspace/curate/Curator.java

@@ -15,6 +15,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.dspace.content.Collection;
 import org.dspace.content.Community;
@@ -69,16 +70,12 @@ public class Curator {
         INTERACTIVE, BATCH, ANY
     }
 
-    ;
-
     // transaction scopes
     public static enum TxScope {
         OBJECT, CURATION, OPEN
     }
 
-    ;
+    private static final Logger log = LogManager.getLogger();
-
-    private static final Logger log = org.apache.logging.log4j.LogManager.getLogger(Curator.class);
 
     protected static final ThreadLocal<Context> curationCtx = new ThreadLocal<>();
 
@@ -86,7 +83,7 @@ public class Curator {
     protected Map<String, TaskRunner> trMap = new HashMap<>();
     protected List<String> perfList = new ArrayList<>();
     protected TaskQueue taskQ = null;
-    protected String reporter = null;
+    protected Appendable reporter = null;
     protected Invoked iMode = null;
     protected TaskResolver resolver = new TaskResolver();
     protected TxScope txScope = TxScope.OPEN;
@@ -193,7 +190,7 @@ public class Curator {
      *                 causes reporting to standard out.
      * @return return self (Curator instance) with reporter set
      */
-    public Curator setReporter(String reporter) {
+    public Curator setReporter(Appendable reporter) {
         this.reporter = reporter;
         return this;
     }
@@ -346,9 +343,10 @@ public class Curator {
      * @param message the message to output to the reporting stream.
      */
     public void report(String message) {
-        // Stub for now
+        try {
-        if ("-".equals(reporter)) {
+            reporter.append(message);
-            System.out.println(message);
+        } catch (IOException ex) {
+            log.error("Task reporting failure", ex);
         }
     }
 

dspace-api/src/main/java/org/dspace/curate/FileReporter.java

@@ -0,0 +1,88 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE and NOTICE files at the root of the source
+ * tree and available online at
+ *
+ * http://www.dspace.org/license/
+ */
+
+package org.dspace.curate;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.Writer;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.GregorianCalendar;
+
+import org.dspace.services.ConfigurationService;
+import org.dspace.utils.DSpace;
+
+/**
+ * Save a curation report to a unique file in the reports directory.
+ * Reports are named by the date and time of day, for example:
+ * "curation-20180916T113903045.report".
+ *
+ * @author mhwood
+ */
+public class FileReporter
+        implements Reporter {
+    private final Writer writer;
+
+    /**
+     * Open a writer to a file in a directory named by the configuration
+     * property {@code report.dir}, or in {@code [DSpace]/reports} if not
+     * configured.
+     *
+     * @throws IOException if there is a problem with the file path.
+     */
+    public FileReporter()
+            throws IOException {
+        // Calculate a unique(?) file name.
+        Date now = GregorianCalendar.getInstance().getTime();
+        SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd'T'hhmmssSSS");
+        String filename = String.format("curation-%s.report", sdf.format(now));
+
+        // Build a path to the directory which is to receive the file.
+        ConfigurationService cfg = new DSpace().getConfigurationService();
+        String reportDir = cfg.getProperty("report.dir");
+        Path reportPath;
+        if (null == reportDir) {
+            reportPath = Paths.get(cfg.getProperty("dspace.dir"),
+                    "reports",
+                    filename);
+        } else {
+            reportPath = Paths.get(reportDir, filename);
+        }
+
+        // Open the file.
+        writer = new FileWriter(reportPath.toFile());
+    }
+
+    @Override
+    public Appendable append(CharSequence cs)
+            throws IOException {
+        writer.append(cs);
+        return this;
+    }
+
+    @Override
+    public Appendable append(CharSequence cs, int i, int i1)
+            throws IOException {
+        writer.append(cs, i, i1);
+        return this;
+    }
+
+    @Override
+    public Appendable append(char c) throws IOException {
+        writer.append(c);
+        return this;
+    }
+
+    @Override
+    public void close() throws Exception {
+        writer.close();
+    }
+}

dspace-api/src/main/java/org/dspace/curate/LogReporter.java

@@ -0,0 +1,62 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE and NOTICE files at the root of the source
+ * tree and available online at
+ *
+ * http://www.dspace.org/license/
+ */
+
+package org.dspace.curate;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Write curation report records through the logging framework.
+ * Whole lines (strings ending in '\n') are written to the log category "curation".
+ * Any partial line is flushed when the reporter is {@code close()}d.
+ *
+ * @author mhwood
+ */
+public class LogReporter
+        implements Reporter {
+    private static final Logger LOG = LoggerFactory.getLogger("curation");
+    private final StringBuilder buffer = new StringBuilder();
+
+    @Override
+    public Appendable append(CharSequence cs)
+            throws IOException {
+        for (int pos = 0; pos < cs.length(); pos++) {
+            char c = cs.charAt(pos);
+            if (c == '\n') {
+                LOG.info(buffer.toString());
+                buffer.delete(0, buffer.length()); // Clear the buffer
+            } else {
+                buffer.append(c);
+            }
+        }
+        return this;
+    }
+
+    @Override
+    public Appendable append(CharSequence cs, int i, int i1)
+            throws IOException {
+        return append(cs.subSequence(i, i1));
+    }
+
+    @Override
+    public Appendable append(char c)
+            throws IOException {
+        return append(String.valueOf(c));
+    }
+
+    @Override
+    public void close()
+            throws Exception {
+        if (buffer.length() > 0) {
+            LOG.info(buffer.toString());
+        }
+    }
+}

dspace-api/src/main/java/org/dspace/curate/Reporter.java

@@ -0,0 +1,18 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE and NOTICE files at the root of the source
+ * tree and available online at
+ *
+ * http://www.dspace.org/license/
+ */
+
+package org.dspace.curate;
+
+/**
+ * A marker interface needed to make curation reporter classes into plugins.
+ *
+ * @author mhwood
+ */
+public interface Reporter
+        extends Appendable, AutoCloseable {
+}

dspace-api/src/main/java/org/dspace/curate/WorkflowCuratorServiceImpl.java

@@ -30,6 +30,8 @@ import org.dspace.content.Item;
 import org.dspace.content.service.CollectionService;
 import org.dspace.core.Context;
 import org.dspace.core.LogManager;
+import org.dspace.core.factory.CoreServiceFactory;
+import org.dspace.core.service.PluginService;
 import org.dspace.curate.service.WorkflowCuratorService;
 import org.dspace.eperson.EPerson;
 import org.dspace.eperson.Group;
@@ -56,9 +58,10 @@ import org.springframework.beans.factory.annotation.Autowired;
 public class WorkflowCuratorServiceImpl implements WorkflowCuratorService {
 
     /**
-     * log4j logger
+     * Logging category
      */
-    private Logger log = org.apache.logging.log4j.LogManager.getLogger(WorkflowCuratorServiceImpl.class);
+    private static final Logger log
+        = org.apache.logging.log4j.LogManager.getLogger();
 
     protected Map<String, TaskSet> tsMap = new HashMap<String, TaskSet>();
 
@@ -118,6 +121,7 @@ public class WorkflowCuratorServiceImpl implements WorkflowCuratorService {
             Curator curator = new Curator();
             // are we going to perform, or just put on queue?
             if (step.queue != null) {
+                // The queue runner will call setReporter
                 for (Task task : step.tasks) {
                     curator.addTask(task.name);
                 }
@@ -125,7 +129,18 @@ public class WorkflowCuratorServiceImpl implements WorkflowCuratorService {
                 basicWorkflowItemService.update(c, wfi);
                 return false;
             } else {
-                return curate(curator, c, wfi);
+                PluginService plugins = CoreServiceFactory.getInstance()
+                        .getPluginService();
+                try (Reporter reporter
+                        = (Reporter) plugins
+                        .getSinglePlugin(Reporter.class);) {
+                    curator.setReporter(reporter);
+                    boolean status = curate(curator, c, wfi);
+                    reporter.close();
+                    return status;
+                } catch (Exception e) {
+                    log.error("Failed to close report", e);
+                }
             }
         }
         return true;

dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseUtils.java

@@ -10,6 +10,7 @@ package org.dspace.storage.rdbms;
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
@@ -23,6 +24,7 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import javax.sql.DataSource;
 
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.Logger;
 import org.dspace.core.Context;
@@ -88,7 +90,8 @@ public class DatabaseUtils {
         // Usage checks
         if (argv.length < 1) {
             System.out.println("\nDatabase action argument is missing.");
-            System.out.println("Valid actions: 'test', 'info', 'migrate', 'repair', 'validate' or 'clean'");
+            System.out.println("Valid actions: 'test', 'info', 'migrate', 'repair', 'validate', " +
+                    "'update-sequences' or 'clean'");
             System.out.println("\nOr, type 'database help' for more information.\n");
             System.exit(1);
         }
@@ -328,23 +331,48 @@ public class DatabaseUtils {
                     e.printStackTrace();
                     System.exit(1);
                 }
+            } else if (argv[0].equalsIgnoreCase("update-sequences")) {
+                try (Connection connection = dataSource.getConnection()) {
+                    String dbType = getDbType(connection);
+                    String sqlfile = "org/dspace/storage/rdbms/sqlmigration/" + dbType +
+                            "/update-sequences.sql";
+                    InputStream sqlstream = DatabaseUtils.class.getClassLoader().getResourceAsStream(sqlfile);
+                    if (sqlstream != null) {
+                        String s = IOUtils.toString(sqlstream, "UTF-8");
+                        if (!s.isEmpty()) {
+                            System.out.println("Running " + sqlfile);
+                            connection.createStatement().execute(s);
+                            System.out.println("update-sequences complete");
+                        } else {
+                            System.err.println(sqlfile + " contains no SQL to execute");
+                        }
+                    } else {
+                        System.err.println(sqlfile + " not found");
+                    }
+                }
             } else {
                 System.out.println("\nUsage: database [action]");
-                System.out.println("Valid actions: 'test', 'info', 'migrate', 'repair' or 'clean'");
+                System.out.println("Valid actions: 'test', 'info', 'migrate', 'repair', " +
+                    "'update-sequences' or 'clean'");
                 System.out.println(
-                    " - test          = Performs a test connection to database to validate connection settings");
+                    " - test             = Performs a test connection to database to " +
+                    "validate connection settings");
                 System.out.println(
                     " - info / status    = Describe basic info/status about database, including validating the " +
                     "compatibility of this database");
-                System.out.println(" - migrate       = Migrate the database to the latest version");
                 System.out.println(
-                    " - repair        = Attempt to repair any previously failed database migrations or checksum " +
+                    " - migrate          = Migrate the database to the latest version");
-                        "mismatches (via Flyway repair)");
+                System.out.println(
+                    " - repair           = Attempt to repair any previously failed database " +
+                    "migrations or checksum mismatches (via Flyway repair)");
                 System.out.println(
                     " - validate         = Validate current database's migration status (via Flyway validate), " +
                     "validating all migration checksums.");
                 System.out.println(
-                    " - clean         = DESTROY all data and tables in database (WARNING there is no going back!). " +
+                    " - update-sequences = Update database sequences after running AIP ingest.");
+                System.out.println(
+                    " - clean            = DESTROY all data and tables in database " +
+                    "(WARNING there is no going back!). " +
                     "Requires 'db.cleanDisabled=false' setting in config.");
                 System.out.println("");
                 System.exit(0);

dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/oracle/update-sequences.sql

@@ -0,0 +1,79 @@
+--
+-- The contents of this file are subject to the license and copyright
+-- detailed in the LICENSE and NOTICE files at the root of the source
+-- tree and available online at
+--
+-- http://www.dspace.org/license/
+--
+
+-- SQL code to update the ID (primary key) generating sequences, if some
+-- import operation has set explicit IDs.
+--
+-- Sequences are used to generate IDs for new rows in the database.  If a
+-- bulk import operation, such as an SQL dump, specifies primary keys for
+-- imported data explicitly, the sequences are out of sync and need updating.
+-- This SQL code does just that.
+--
+-- This should rarely be needed; any bulk import should be performed using the
+-- org.dspace.content API which is safe to use concurrently and in multiple
+-- JVMs.  The SQL code below will typically only be required after a direct
+-- SQL data dump from a backup or somesuch.
+
+-- The 'updateseq' procedure was derived from incseq.sql found at:
+-- http://www.akadia.com/services/scripts/incseq.sql
+
+DECLARE
+ PROCEDURE updateseq ( seq IN VARCHAR,
+                       tbl IN VARCHAR,
+                       attr IN VARCHAR,
+                       cond IN VARCHAR DEFAULT '' ) IS
+   curr NUMBER := 0;
+   BEGIN
+     EXECUTE IMMEDIATE 'SELECT max(' || attr
+             || ') FROM ' || tbl
+             || ' ' || cond
+             INTO curr;
+     curr := curr + 1;
+     EXECUTE IMMEDIATE 'DROP SEQUENCE ' || seq;
+     EXECUTE IMMEDIATE 'CREATE SEQUENCE '
+             || seq
+             || ' START WITH '
+             || NVL(curr, 1);
+   END updateseq;
+
+BEGIN
+  updateseq('bitstreamformatregistry_seq', 'bitstreamformatregistry',
+            'bitstream_format_id');
+  updateseq('fileextension_seq', 'fileextension', 'file_extension_id');
+  updateseq('resourcepolicy_seq', 'resourcepolicy', 'policy_id');
+  updateseq('workspaceitem_seq', 'workspaceitem', 'workspace_item_id');
+  updateseq('workflowitem_seq', 'workflowitem', 'workflow_id');
+  updateseq('tasklistitem_seq', 'tasklistitem', 'tasklist_id');
+  updateseq('registrationdata_seq', 'registrationdata',
+            'registrationdata_id');
+  updateseq('subscription_seq', 'subscription', 'subscription_id');
+  updateseq('metadatafieldregistry_seq', 'metadatafieldregistry',
+            'metadata_field_id');
+  updateseq('metadatavalue_seq', 'metadatavalue', 'metadata_value_id');
+  updateseq('metadataschemaregistry_seq', 'metadataschemaregistry',
+            'metadata_schema_id');
+  updateseq('harvested_collection_seq', 'harvested_collection', 'id');
+  updateseq('harvested_item_seq', 'harvested_item', 'id');
+  updateseq('webapp_seq', 'webapp', 'webapp_id');
+  updateseq('requestitem_seq', 'requestitem', 'requestitem_id');
+  updateseq('handle_id_seq', 'handle', 'handle_id');
+
+  -- Handle Sequence is a special case.  Since Handles minted by DSpace
+  -- use the 'handle_seq', we need to ensure the next assigned handle
+  -- will *always* be unique.  So, 'handle_seq' always needs to be set
+  -- to the value of the *largest* handle suffix.  That way when the
+  -- next handle is assigned, it will use the next largest number. This
+  -- query does the following:
+  --   For all 'handle' values which have a number in their suffix
+  --   (after '/'), find the maximum suffix value, convert it to a
+  --   number, and set the 'handle_seq' to start at the next value (see
+  --   updateseq above for more).
+  updateseq('handle_seq', 'handle',
+            q'{to_number(regexp_replace(handle, '.*/', ''), '999999999999')}',
+            q'{WHERE REGEXP_LIKE(handle, '^.*/[0123456789]*$')}');
+END;

dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/postgres/README.md

@@ -17,4 +17,16 @@ not realize you manually ran one or more scripts.
 
 Please see the Flyway Documentation for more information: http://flywaydb.org/
 
+## Using the update-sequences.sql script
+
+The `update-sequences.sql` script in this directory may still be used to update
+your internal database counts if you feel they have gotten out of "sync". This
+may sometimes occur after large restores of content (e.g. when using the DSpace
+[AIP Backup and Restore](https://wiki.duraspace.org/display/DSDOC5x/AIP+Backup+and+Restore) 
+feature).
+
+This `update-sequences.sql` script can be executed by running 
+"dspace database update-sequences". It will not harm your 
+database (or its contents) in any way. It just ensures all database counts (i.e.
+sequences) are properly set to the next available value.
 

dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/postgres/update-sequences.sql

@@ -1,35 +1,10 @@
 --
--- update-sequences.sql
+-- The contents of this file are subject to the license and copyright
+-- detailed in the LICENSE and NOTICE files at the root of the source
+-- tree and available online at
 --
--- Copyright (c) 2002-2016, The DSpace Foundation.  All rights reserved.
+-- http://www.dspace.org/license/
 --
--- Redistribution and use in source and binary forms, with or without
--- modification, are permitted provided that the following conditions are
--- met:
--- 
--- - Redistributions of source code must retain the above copyright
--- notice, this list of conditions and the following disclaimer.
--- 
--- - Redistributions in binary form must reproduce the above copyright
--- notice, this list of conditions and the following disclaimer in the
--- documentation and/or other materials provided with the distribution.
--- 
--- Neither the name of the DSpace Foundation nor the names of its
--- contributors may be used to endorse or promote products derived from
--- this software without specific prior written permission.
--- 
--- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
--- ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
--- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
--- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
--- HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
--- INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
--- BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
--- OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
--- ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
--- TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
--- USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
--- DAMAGE.
 
 -- SQL code to update the ID (primary key) generating sequences, if some
 -- import operation has set explicit IDs.

dspace-api/src/test/java/org/dspace/curate/CuratorTest.java

@@ -58,7 +58,7 @@ public class CuratorTest
 
         // Get and configure a Curator.
         Curator instance = new Curator();
-        instance.setReporter("-"); // Send any report to standard out. FIXME when DS-3989 is merged
+        instance.setReporter(System.out); // Send any report to standard out.
         instance.addTask(TASK_NAME);
 
         // Configure the run.

dspace-api/src/test/java/org/dspace/curate/ITCurator.java

@@ -0,0 +1,202 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE and NOTICE files at the root of the source
+ * tree and available online at
+ *
+ * http://www.dspace.org/license/
+ */
+
+package org.dspace.curate;
+
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import org.dspace.AbstractUnitTest;
+import org.dspace.authorize.AuthorizeException;
+import org.dspace.content.Community;
+import org.dspace.content.DSpaceObject;
+import org.dspace.content.Site;
+import org.dspace.content.factory.ContentServiceFactory;
+import org.dspace.services.ConfigurationService;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Drive the Curator and check results.
+ *
+ * @author mhwood
+ */
+public class ITCurator
+        extends AbstractUnitTest {
+    Logger LOG = LoggerFactory.getLogger(ITCurator.class);
+
+    public ITCurator() {
+    }
+
+    @BeforeClass
+    public static void setUpClass() {
+    }
+
+    @AfterClass
+    public static void tearDownClass() {
+    }
+
+    @Before
+    public void setUp() {
+    }
+
+    @After
+    public void tearDown() {
+    }
+
+    /**
+     * The report should contain contributions from all tasks and all curated objects.
+     *
+     * @throws SQLException passed through.
+     * @throws IOException passed through.
+     * @throws AuthorizeException passed through.
+     */
+    @Test
+    public void testCurate_Reporting()
+            throws SQLException, IOException, AuthorizeException {
+        // Configure for testing.
+        ConfigurationService cfg = kernelImpl.getConfigurationService();
+        cfg.setProperty("plugin.named.org.dspace.curate.CurationTask",
+                Task1.class.getName() + " = task1");
+        cfg.addPropertyValue("plugin.named.org.dspace.curate.CurationTask",
+                Task2.class.getName() + " = task2");
+
+        // Create some structure.
+        context.turnOffAuthorisationSystem();
+        Site site = ContentServiceFactory.getInstance()
+                .getSiteService()
+                .findSite(context);
+        Community community = ContentServiceFactory.getInstance()
+                .getCommunityService()
+                .create(null, context);
+
+        // Run some tasks.
+        ListReporter reporter = new ListReporter();
+        Curator curator = new Curator();
+        curator.setReporter(reporter);
+        curator.addTask("task1");
+        curator.addTask("task2");
+        curator.curate(context, site);
+
+        // Validate the results.
+        List<String> report = reporter.getReport();
+        for (String aReport : report) {
+            LOG.info("Report:  {}", aReport);
+        }
+        Pattern pattern;
+        pattern = Pattern.compile(String.format("task1.*%s", site.getHandle()));
+        Assert.assertTrue("A report should mention 'task1' and site's handle",
+                reportMatcher(report, pattern));
+        pattern = Pattern.compile(String.format("task1.*%s", community.getHandle()));
+        Assert.assertTrue("A report should mention 'task1' and the community's handle",
+                reportMatcher(report, pattern));
+        pattern = Pattern.compile(String.format("task2.*%s", site.getHandle()));
+        Assert.assertTrue("A report should mention 'task2' and the Site's handle",
+                reportMatcher(report, pattern));
+        pattern = Pattern.compile(String.format("task2.*%s", community.getHandle()));
+        Assert.assertTrue("A report should mention 'task2' and the community's handle",
+                reportMatcher(report, pattern));
+    }
+
+    /**
+     * Match a collection of strings against a regular expression.\
+     *
+     * @param reports strings to be searched.
+     * @param pattern expression to be matched.
+     * @return true if at least one string matches the expression.
+     */
+    private boolean reportMatcher(List<String> reports, Pattern pattern) {
+        for (String aReport : reports) {
+            if (pattern.matcher(aReport).find()) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Dummy curation task for testing.  Reports how it was invoked.
+     *
+     * @author mhwood
+     */
+    public static class Task1 extends AbstractCurationTask {
+        public Task1() {
+        }
+
+        @Override
+        public int perform(DSpaceObject dso)
+                throws IOException {
+            curator.report(String.format(
+                    "Task1 received 'perform' on taskId '%s' for object '%s'%n",
+                    taskId, dso.getHandle()));
+            return Curator.CURATE_SUCCESS;
+        }
+    }
+
+    /**
+     * Dummy curation task for testing.  Reports how it was invoked.
+     *
+     * @author mhwood
+     */
+    public static class Task2 extends AbstractCurationTask {
+        public Task2() {
+        }
+
+        @Override
+        public int perform(DSpaceObject dso) throws IOException {
+            curator.report(String.format(
+                    "Task2 received 'perform' on taskId '%s' for object '%s'%n",
+                    taskId, dso.getHandle()));
+            return Curator.CURATE_SUCCESS;
+        }
+    }
+
+    /**
+     * Absorb report strings into a sequential collection.
+     */
+    class ListReporter
+            implements Appendable {
+        private final List<String> report = new ArrayList<>();
+
+        /**
+         * Get the content of the report accumulator.
+         * @return accumulated reports.
+         */
+        List<String> getReport() {
+            return report;
+        }
+
+        @Override
+        public Appendable append(CharSequence cs)
+                throws IOException {
+            report.add(cs.toString());
+            return this;
+        }
+
+        @Override
+        public Appendable append(CharSequence cs, int i, int i1)
+                throws IOException {
+            throw new UnsupportedOperationException("Not supported yet.");
+        }
+
+        @Override
+        public Appendable append(char c)
+                throws IOException {
+            throw new UnsupportedOperationException("Not supported yet.");
+        }
+    }
+}

dspace-services/src/main/java/org/dspace/servicemanager/config/DSpaceConfigurationService.java

@@ -346,6 +346,26 @@ public final class DSpaceConfigurationService implements ConfigurationService {
         }
     }
 
+    @Override
+    public synchronized boolean addPropertyValue(String name, Object value) {
+        if (name == null) {
+            throw new IllegalArgumentException("name cannot be null for setting configuration");
+        }
+        if (value == null) {
+            throw new IllegalArgumentException("configuration value may not be null");
+        }
+
+        // If the value is a type of String, trim any leading/trailing spaces before saving it.
+        if (String.class.isInstance(value)) {
+            value = ((String) value).trim();
+        }
+
+        Configuration configuration = getConfiguration();
+        boolean isNew = !configuration.containsKey(name);
+        configuration.addProperty(name, value);
+        return isNew;
+    }
+
     /* (non-Javadoc)
      * @see org.dspace.services.ConfigurationService#setProperty(java.lang.String, java.lang.Object)
      */

dspace-services/src/main/java/org/dspace/services/ConfigurationService.java

@@ -237,6 +237,16 @@ public interface ConfigurationService {
      */
     public boolean hasProperty(String name);
 
+    /**
+     * Add a value to a configuration property.
+     *
+     * @param name the property name.  May not be null.
+     * @param value the property value.  May not be null.
+     * @return true if a new property was created.
+     * @throws IllegalArgumentException if the name or value is null.
+     */
+    public boolean addPropertyValue(String name, Object value);
+
     /**
      * Set a configuration property (setting) in the system.
      * Type is not important here since conversion happens automatically

dspace/config/modules/submission-curation.cfg

@@ -2,7 +2,15 @@
 #------------SUBMISSION CURATION CONFIGURATIONS-----------------#
 #---------------------------------------------------------------#
 # This file contains configuration properties solely relating   #
-# to the scheduling of curation tasks during submission.        #
+# to the scheduling of curation tasks during submission -- that #
+# is:  when tasks are attached to a workflow.                   #
 #---------------------------------------------------------------#
 # Scan for viruses
 submission-curation.virus-scan = false
+
+# Report serializer plugin, to capture submission task reports.
+# Uncomment exactly one, or configure your own.
+# FileReporter writes reports to ${report.dir}/curation-yyyyMMddThhmmssSSS.report
+plugin.single.org.dspace.curate.Reporter = org.dspace.curate.FileReporter
+# LogReporter writes report lines to the DSpace log.
+#plugin.single.org.dspace.curate.Reporter = org.dspace.curate.LogReporter

dspace/etc/oracle/README.md

@@ -1,33 +0,0 @@
-# DSpace Database Now Upgrades Automatically
-
-AS OF DSPACE 5, the DSpace database now upgrades itself AUTOMATICALLY.
-
-Therefore, all `database_schema*.sql` files have been removed. Starting
-with DSpace 4.x -> 5.0 upgrade, you will no longer need to manually run any
-SQL scripts to upgrade your database.
-
-Please see the [5.0 Upgrade Instructions](https://wiki.duraspace.org/display/DSDOC5x/Upgrading+DSpace)
-for more information on upgrading to DSpace 5.
-
-
-## More info on automatic database upgrades
-
-As of DSpace 5.0, we now use [Flyway DB](http://flywaydb.org/) along with the
-SQL scripts embedded in the `dspace-api.jar` to automatically keep your DSpace
-database up-to-date. These scripts are now located in the source code at:
-`[dspace-src]/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/oracle`
-
-As Flyway automates the upgrade process, you should NEVER run these SQL scripts
-manually. For more information, please see the `README.md` in the scripts directory.
-
-## Using the update-sequences.sql script
-
-The `update-sequences.sql` script in this directory may still be used to update
-your internal database counts if you feel they have gotten out of "sync". This
-may sometimes occur after large restores of content (e.g. when using the DSpace
-[AIP Backup and Restore](https://wiki.duraspace.org/display/DSDOC5x/AIP+Backup+and+Restore) 
-feature).
-
-This `update-sequences.sql` script can be run manually. It will not harm your 
-database (or its contents) in any way. It just ensures all database counts (i.e.
-sequences) are properly set to the next available value.

dspace/etc/oracle/update-sequences.sql

@@ -1,75 +0,0 @@
---
--- update-sequences.sql
---
--- Copyright (c) 2002-2016, The DSpace Foundation.  All rights reserved.
--- 
--- Redistribution and use in source and binary forms, with or without
--- modification, are permitted provided that the following conditions are
--- met:
--- 
--- - Redistributions of source code must retain the above copyright
--- notice, this list of conditions and the following disclaimer.
--- 
--- - Redistributions in binary form must reproduce the above copyright
--- notice, this list of conditions and the following disclaimer in the
--- documentation and/or other materials provided with the distribution.
--- 
--- - Neither the name of the DSpace Foundation nor the names of its
--- contributors may be used to endorse or promote products derived from
--- this software without specific prior written permission.
--- 
--- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
--- ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
--- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
--- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
--- HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
--- INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
--- BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
--- OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
--- ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
--- TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
--- USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
--- DAMAGE.
-
--- SQL code to update the ID (primary key) generating sequences, if some
--- import operation has set explicit IDs.
---
--- Sequences are used to generate IDs for new rows in the database.  If a
--- bulk import operation, such as an SQL dump, specifies primary keys for
--- imported data explicitly, the sequences are out of sync and need updating.
--- This SQL code does just that.
---
--- This should rarely be needed; any bulk import should be performed using the
--- org.dspace.content API which is safe to use concurrently and in multiple
--- JVMs.  The SQL code below will typically only be required after a direct
--- SQL data dump from a backup or somesuch.
-
--- Depends on being run from sqlplus with incseq.sql in the current path
--- you can find incseq.sql at: http://www.akadia.com/services/scripts/incseq.sql
--- Here that script was renamed to updateseq.sql.
-
-@updateseq.sql bitstreamformatregistry_seq bitstreamformatregistry bitstream_format_id ""
-@updateseq.sql fileextension_seq fileextension file_extension_id ""
-@updateseq.sql resourcepolicy_seq resourcepolicy policy_id ""
-@updateseq.sql workspaceitem_seq workspaceitem workspace_item_id ""
-@updateseq.sql workflowitem_seq workflowitem workflow_id ""
-@updateseq.sql tasklistitem_seq tasklistitem tasklist_id ""
-@updateseq.sql registrationdata_seq registrationdata registrationdata_id ""
-@updateseq.sql subscription_seq subscription subscription_id ""
-@updateseq.sql metadatafieldregistry_seq metadatafieldregistry metadata_field_id ""
-@updateseq.sql metadatavalue_seq metadatavalue metadata_value_id ""
-@updateseq.sql metadataschemaregistry_seq metadataschemaregistry metadata_schema_id ""
-@updateseq.sql harvested_collection_seq harvested_collection id ""
-@updateseq.sql harvested_item_seq harvested_item id ""
-@updateseq.sql webapp_seq webapp webapp_id ""
-@updateseq.sql requestitem_seq requestitem requestitem_id ""
-@updateseq.sql handle_id_seq handle handle_id ""
-
--- Handle Sequence is a special case.  Since Handles minted by DSpace use the 'handle_seq',
--- we need to ensure the next assigned handle will *always* be unique.  So, 'handle_seq'
--- always needs to be set to the value of the *largest* handle suffix.  That way when the
--- next handle is assigned, it will use the next largest number. This query does the following:
---  For all 'handle' values which have a number in their suffix (after '/'), find the maximum
---  suffix value, convert it to a number, and set the 'handle_seq' to start at the next value
--- (see updateseq.sql script for more)
-@updateseq.sql handle_seq handle "to_number(regexp_replace(handle, '.*/', ''), '999999999999')" "WHERE REGEXP_LIKE(handle, '^.*/[0123456789]*$')"

dspace/etc/oracle/updateseq.sql

@@ -1,30 +0,0 @@
--- #############################################################################################
---
--- %Purpose: Set a sequence to the max value of a given attribute
---
--- #############################################################################################
---
--- Paramters:
---   1: sequence name
---   2: table name
---   3: attribute name
---
--- Sample usage:
---   @updateseq.sql my_sequence my_table my_attribute where-clause
---
---------------------------------------------------------------------------------
---
-SET SERVEROUTPUT ON SIZE 1000000;
---
-DECLARE
-  curr  NUMBER := 0;
-BEGIN
-  SELECT max(&3) INTO curr FROM &2 &4;
-
-  curr := curr + 1;
-
-  EXECUTE IMMEDIATE 'DROP SEQUENCE &1';
-
-  EXECUTE IMMEDIATE 'CREATE SEQUENCE &1 START WITH ' || NVL(curr,1);
-END;
-/

dspace/etc/postgres/README.md

@@ -1,33 +0,0 @@
-# DSpace Database Now Upgrades Automatically
-
-AS OF DSPACE 5, the DSpace database now upgrades itself AUTOMATICALLY.
-
-Therefore, all `database_schema*.sql` files have been removed. Starting
-with DSpace 4.x -> 5.0 upgrade, you will no longer need to manually run any
-SQL scripts to upgrade your database.
-
-Please see the [5.0 Upgrade Instructions](https://wiki.duraspace.org/display/DSDOC5x/Upgrading+DSpace)
-for more information on upgrading to DSpace 5.
-
-
-## More info on automatic database upgrades
-
-As of DSpace 5.0, we now use [Flyway DB](http://flywaydb.org/) along with the
-SQL scripts embedded in the `dspace-api.jar` to automatically keep your DSpace
-database up-to-date. These scripts are now located in the source code at:
-`[dspace-src]/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/postgres`
-
-As Flyway automates the upgrade process, you should NEVER run these SQL scripts
-manually. For more information, please see the `README.md` in the scripts directory.
-
-## Using the update-sequences.sql script
-
-The `update-sequences.sql` script in this directory may still be used to update
-your internal database counts if you feel they have gotten out of "sync". This
-may sometimes occur after large restores of content (e.g. when using the DSpace
-[AIP Backup and Restore](https://wiki.duraspace.org/display/DSDOC5x/AIP+Backup+and+Restore) 
-feature).
-
-This `update-sequences.sql` script can be run manually. It will not harm your 
-database (or its contents) in any way. It just ensures all database counts (i.e.
-sequences) are properly set to the next available value.

dspace/src/main/config/build.xml

@@ -569,16 +569,6 @@ Common usage:
             <fileset dir="lib" />
         </copy>
 
-        <move todir="${dspace.dir}/etc.bak-${build.date}" failonerror="no">
-            <fileset dir="${dspace.dir}/etc">
-                <include name="**/*" />
-            </fileset>
-        </move>
-
-        <copy todir="${dspace.dir}/etc" preservelastmodified="true">
-            <fileset dir="etc" />
-        </copy>
-
     	<copy todir="${dspace.dir}/solr" preservelastmodified="true">
     	     <fileset dir="solr"/>
     	</copy>
@@ -593,10 +583,6 @@ Common usage:
 
  ${dspace.dir}/lib.bak-${build.date}
 
- ${dspace.dir}/etc was backed up to
-
- ${dspace.dir}/etc.bak-${build.date}
-
  Please review these directories and delete if no longer needed.
 ====================================================================
         </echo>
@@ -728,8 +714,6 @@ Common usage:
 
         <mkdir dir="${dspace.dir}/lib" />
 
-        <mkdir dir="${dspace.dir}/etc" />
-
         <mkdir dir="${dspace.dir}/webapps" />
 
         <mkdir dir="${dspace.dir}/triplestore" />
@@ -821,17 +805,6 @@ Common usage:
             <fileset dir="lib" />
         </copy>
 
-        <!-- NB: no regular use is made of etc/ files in an installed
-        - system, so this step is 'deprecated', and will eventually
-        - be removed.  -->
-        <delete failonerror="no">
-            <fileset dir="${dspace.dir}/etc" includes="**/*" />
-        </delete>
-
-        <copy todir="${dspace.dir}/etc" preservelastmodified="true">
-            <fileset dir="etc" />
-        </copy>
-
     	<copy todir="${dspace.dir}/solr" preservelastmodified="true">
     	    <fileset dir="solr" />
     	</copy>