[Task 66428] Applied feedback to the MetadataImport and Export scripts and framework

dspace-api/src/main/java/org/dspace/app/bulkedit/DSpaceCSV.java

@@ -8,12 +8,8 @@
 package org.dspace.app.bulkedit;
 
 import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.FileOutputStream;
-import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -623,24 +619,6 @@ public class DSpaceCSV implements Serializable {
         return csvLines;
     }
 
-    /**
-     * Save the CSV file to the given filename
-     *
-     * @param filename The filename to save the CSV file to
-     * @throws IOException Thrown if an error occurs when writing the file
-     */
-    public final void save(String filename) throws IOException {
-        // Save the file
-        BufferedWriter out = new BufferedWriter(
-            new OutputStreamWriter(
-                new FileOutputStream(filename), "UTF-8"));
-        for (String csvLine : getCSVLinesAsStringArray()) {
-            out.write(csvLine + "\n");
-        }
-        out.flush();
-        out.close();
-    }
-
     public InputStream getInputStream() {
         StringBuilder stringBuilder = new StringBuilder();
         for (String csvLine : getCSVLinesAsStringArray()) {

dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExport.java

@@ -21,7 +21,7 @@ import org.springframework.beans.factory.annotation.Autowired;
  */
 public class MetadataExport extends DSpaceRunnable {
 
-    private Context c = null;
+    private Context context = null;
     private boolean help = false;
     private String filename = null;
     private String handle = null;
@@ -31,43 +31,46 @@ public class MetadataExport extends DSpaceRunnable {
     @Autowired
     private MetadataExportService metadataExportService;
 
-    protected Context context;
-
     private MetadataExport() {
-        Options options = constructOptions();
+        this.options = constructOptions();
-        this.options = options;
     }
 
     private Options constructOptions() {
         Options options = new Options();
 
         options.addOption("i", "id", true, "ID or handle of thing to export (item, collection, or community)");
+        options.getOption("i").setType(String.class);
         options.addOption("f", "file", true, "destination where you want file written");
+        options.getOption("f").setType(String.class);
         options.getOption("f").setRequired(true);
         options.addOption("a", "all", false,
                           "include all metadata fields that are not normally changed (e.g. provenance)");
+        options.getOption("a").setType(boolean.class);
         options.addOption("h", "help", false, "help");
+        options.getOption("h").setType(boolean.class);
+
 
         return options;
     }
 
     public void internalRun() throws Exception {
         if (help) {
-            handler.logInfo("\nfull export: metadataexport -f filename");
+            handler.logInfo("\nfull export: metadata-export -f filename");
-            handler.logInfo("partial export: metadataexport -i handle -f filename");
+            handler.logInfo("partial export: metadata-export -i handle -f filename");
             printHelp();
             return;
         }
 
-        DSpaceCSV dSpaceCSV = metadataExportService.handleExport(c, exportAllItems, exportAllMetadata, handle);
+        DSpaceCSV dSpaceCSV = metadataExportService.handleExport(context, exportAllItems, exportAllMetadata, handle,
-        handler.writeFilestream(c, filename, dSpaceCSV.getInputStream(), "exportCSV");
+                                                                 handler);
-        c.restoreAuthSystemState();
+        handler.writeFilestream(context, filename, dSpaceCSV.getInputStream(), "exportCSV");
-        c.complete();
+        context.restoreAuthSystemState();
+        context.complete();
     }
 
     public void setup() throws ParseException {
-        c = new Context();
+        context = new Context();
-        c.turnOffAuthorisationSystem();
+        context.turnOffAuthorisationSystem();
 
         if (commandLine.hasOption('h')) {
             help = true;

dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImport.java

@@ -21,7 +21,6 @@ import java.util.Set;
 import java.util.UUID;
 import javax.annotation.Nullable;
 
-import org.apache.commons.cli.HelpFormatter;
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
 import org.apache.commons.lang3.StringUtils;
@@ -59,6 +58,7 @@ import org.dspace.eperson.EPerson;
 import org.dspace.eperson.factory.EPersonServiceFactory;
 import org.dspace.handle.service.HandleService;
 import org.dspace.scripts.DSpaceRunnable;
+import org.dspace.scripts.handler.DSpaceRunnableHandler;
 import org.dspace.workflow.WorkflowItem;
 import org.dspace.workflow.WorkflowService;
 import org.dspace.workflow.factory.WorkflowServiceFactory;
@@ -219,7 +219,7 @@ public class MetadataImport extends DSpaceRunnable implements InitializingBean {
                 try {
                     // Ask the user if they want to make the changes
                     handler.logInfo("\n" + changeCounter + " item(s) will be changed\n");
-                    change = handler.getUserValidation();
+                    change = determineChange(handler);
 
                 } catch (IOException ioe) {
                     throw new IOException("Error: " + ioe.getMessage() + ", No changes have been made", ioe);
@@ -256,6 +256,10 @@ public class MetadataImport extends DSpaceRunnable implements InitializingBean {
 
     }
 
+    protected boolean determineChange(DSpaceRunnableHandler handler) throws IOException {
+        return true;
+    }
+
     public void setup() throws ParseException {
         useTemplate = false;
         filename = null;
@@ -322,7 +326,7 @@ public class MetadataImport extends DSpaceRunnable implements InitializingBean {
         change = false;
     }
 
-    private MetadataImport() {
+    public MetadataImport() {
         Options options = constructOptions();
         this.options = options;
     }
@@ -1230,20 +1234,6 @@ public class MetadataImport extends DSpaceRunnable implements InitializingBean {
         return in.replaceAll("\r\n", "").replaceAll("\n", "").trim();
     }
 
-    /**
-     * Print the help message
-     *
-     * @param options  The command line options the user gave
-     * @param exitCode the system exit code to use
-     */
-    private static void printHelp(Options options, int exitCode) {
-        // print the help message
-        HelpFormatter myhelp = new HelpFormatter();
-        myhelp.printHelp("MetatadataImport\n", options);
-        System.out.println("\nmetadataimport: MetadataImport -f filename");
-        System.exit(exitCode);
-    }
-
     /**
      * Display the changes that have been detected, or that have been made
      *

dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImportCLI.java

@@ -0,0 +1,29 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE and NOTICE files at the root of the source
+ * tree and available online at
+ *
+ * http://www.dspace.org/license/
+ */
+package org.dspace.app.bulkedit;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+import org.dspace.scripts.handler.DSpaceRunnableHandler;
+
+public class MetadataImportCLI extends MetadataImport {
+
+    @Override
+    protected boolean determineChange(DSpaceRunnableHandler handler) throws IOException {
+        handler.logInfo("Do you want to make these changes? [y/n] ");
+        try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(System.in))) {
+            String yn = bufferedReader.readLine();
+            if ("y".equalsIgnoreCase(yn)) {
+                return true;
+            }
+            return false;
+        }
+    }
+}

dspace-api/src/main/java/org/dspace/content/MetadataExportServiceImpl.java

@@ -13,28 +13,27 @@ import java.util.Iterator;
 import java.util.List;
 
 import com.google.common.collect.Iterators;
-import org.apache.logging.log4j.Logger;
 import org.dspace.app.bulkedit.DSpaceCSV;
 import org.dspace.content.service.ItemService;
 import org.dspace.content.service.MetadataExportService;
 import org.dspace.core.Constants;
 import org.dspace.core.Context;
 import org.dspace.handle.factory.HandleServiceFactory;
+import org.dspace.scripts.handler.DSpaceRunnableHandler;
 import org.springframework.beans.factory.annotation.Autowired;
 
 public class MetadataExportServiceImpl implements MetadataExportService {
 
-    private static final Logger log = org.apache.logging.log4j.LogManager.getLogger(MetadataExportServiceImpl.class);
-
     @Autowired
     private ItemService itemService;
 
-    public DSpaceCSV handleExport(Context context, boolean exportAllItems, boolean exportAllMetadata, String handle)
+    @Override
-        throws Exception {
+    public DSpaceCSV handleExport(Context context, boolean exportAllItems, boolean exportAllMetadata, String handle,
+                                  DSpaceRunnableHandler handler) throws Exception {
         Iterator<Item> toExport = null;
 
         if (!exportAllItems) {
-            log.info("Exporting whole repository WARNING: May take some time!");
+            handler.logInfo("Exporting whole repository WARNING: May take some time!");
             toExport = itemService.findAll(context);
         } else {
             DSpaceObject dso = HandleServiceFactory.getInstance().getHandleService().resolveToObject(context, handle);
@@ -44,16 +43,16 @@ public class MetadataExportServiceImpl implements MetadataExportService {
             }
 
             if (dso.getType() == Constants.ITEM) {
-                log.info("Exporting item '" + dso.getName() + "' (" + handle + ")");
+                handler.logInfo("Exporting item '" + dso.getName() + "' (" + handle + ")");
                 List<Item> item = new ArrayList<>();
                 item.add((Item) dso);
                 toExport = item.iterator();
             } else if (dso.getType() == Constants.COLLECTION) {
-                log.info("Exporting collection '" + dso.getName() + "' (" + handle + ")");
+                handler.logInfo("Exporting collection '" + dso.getName() + "' (" + handle + ")");
                 Collection collection = (Collection) dso;
                 toExport = itemService.findByCollection(context, collection);
             } else if (dso.getType() == Constants.COMMUNITY) {
-                log.info("Exporting community '" + dso.getName() + "' (" + handle + ")");
+                handler.logInfo("Exporting community '" + dso.getName() + "' (" + handle + ")");
                 toExport = buildFromCommunity(context, (Community) dso);
             } else {
                 throw new IllegalArgumentException("Error identifying '" + handle + "'");
@@ -64,11 +63,7 @@ public class MetadataExportServiceImpl implements MetadataExportService {
         return csv;
     }
 
-    /**
+    @Override
-     * Run the export
-     *
-     * @return the exported CSV lines
-     */
     public DSpaceCSV export(Context context, Iterator<Item> toExport, boolean exportAll) throws Exception {
         Context.Mode originalMode = context.getCurrentMode();
         context.setMode(Context.Mode.READ_ONLY);
@@ -86,6 +81,7 @@ public class MetadataExportServiceImpl implements MetadataExportService {
         return csv;
     }
 
+    @Override
     public DSpaceCSV export(Context context, Community community, boolean exportAll) throws Exception {
         return export(context, buildFromCommunity(context, community), exportAll);
     }

dspace-api/src/main/java/org/dspace/content/service/MetadataExportService.java

@@ -13,14 +13,46 @@ import org.dspace.app.bulkedit.DSpaceCSV;
 import org.dspace.content.Community;
 import org.dspace.content.Item;
 import org.dspace.core.Context;
+import org.dspace.scripts.handler.DSpaceRunnableHandler;
 
+/**
+ * This is the interface to be implemented by a Service that deals with the exporting of Metadata
+ */
 public interface MetadataExportService {
 
-    public DSpaceCSV handleExport(Context context, boolean exportAllItems, boolean exportAllMetadata, String handle)
+    /**
-        throws Exception;
+     * This method will export DSpaceObject objects depending on the parameters it gets. It can export all the items
+     * in the repository, all the items in a community, all the items in a collection or a specific item. The latter
+     * three are specified by the handle parameter. The entire repository can be exported by defining the
+     * exportAllItems parameter as true
+     * @param context           The relevant DSpace context
+     * @param exportAllItems    A boolean indicating whether or not the entire repository should be exported
+     * @param exportAllMetadata Defines if all metadata should be exported or only the allowed ones
+     * @param handle            The handle for the DSpaceObject to be exported, can be a Community, Collection or Item
+     * @return                  A DSpaceCSV object containing the exported information
+     * @throws Exception        If something goes wrong
+     */
+    public DSpaceCSV handleExport(Context context, boolean exportAllItems, boolean exportAllMetadata,
+                                  String handle, DSpaceRunnableHandler dSpaceRunnableHandler) throws Exception;
 
+    /**
+     * This method will export all the Items in the given toExport iterator to a DSpaceCSV
+     * @param context       The relevant DSpace context
+     * @param toExport      The iterator containing the items to export
+     * @param exportAll     Defines if all metadata should be exported or only the allowed ones
+     * @return              A DSpaceCSV object containing the exported information
+     * @throws Exception    If something goes wrong
+     */
     public DSpaceCSV export(Context context, Iterator<Item> toExport, boolean exportAll) throws Exception;
 
+    /**
+     * This method will export all the Items within the given Community to a DSpaceCSV
+     * @param context       The relevant DSpace context
+     * @param community     The Community that contains the Items to be exported
+     * @param exportAll     Defines if all metadata should be exported or only the allowed ones
+     * @return              A DSpaceCSV object containing the exported information
+     * @throws Exception    If something goes wrong
+     */
     public DSpaceCSV export(Context context, Community community, boolean exportAll) throws Exception;
 
 }

dspace-api/src/main/java/org/dspace/scripts/DSpaceRunnable.java

@@ -77,6 +77,11 @@ public abstract class DSpaceRunnable implements Runnable {
         return options;
     }
 
+    /**
+     * This method will traverse all the options and it'll grab options defined as an InputStream type to then save
+     * the filename specified by that option in a list of Strings that'll be returned in the end
+     * @return  The list of Strings representing filenames from the options given to the script
+     */
     public List<String> getFileNamesFromInputStreamOptions() {
         List<String> fileNames = new LinkedList<>();
 

dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java

@@ -184,6 +184,7 @@ public class ProcessServiceImpl implements ProcessService {
         return null;
     }
 
+    @Override
     public List<Bitstream> getBitstreams(Context context, Process process, String type) {
         List<Bitstream> allBitstreams = process.getBitstreams();
 

dspace-api/src/main/java/org/dspace/scripts/handler/DSpaceRunnableHandler.java

@@ -83,11 +83,27 @@ public interface DSpaceRunnableHandler {
      */
     public void printHelp(Options options, String name);
 
+    /**
+     * This method will grab the InputStream for the file defined by the given file name. The exact implementation will
+     * differ based on whether it's a REST call or CommandLine call. The REST Call will look for Bitstreams in the
+     * Database whereas the CommandLine call will look on the filesystem
+     * @param context       The relevant DSpace context
+     * @param fileName      The filename for the file that holds the InputStream
+     * @return              The InputStream for the file defined by the given file name
+     * @throws IOException  If something goes wrong
+     * @throws AuthorizeException   If something goes wrong
+     */
     public InputStream getFileStream(Context context, String fileName) throws IOException, AuthorizeException;
 
+    /**
+     * This method will write the InputStream to either a file on the filesystem or a bitstream in the database
+     * depending on whether it's coming from a CommandLine call or REST call respectively
+     * @param context       The relevant DSpace context
+     * @param fileName      The filename
+     * @param inputStream   The inputstream to be written
+     * @param type          The type of the file
+     * @throws IOException  If something goes wrong
+     */
     public void writeFilestream(Context context, String fileName, InputStream inputStream, String type)
         throws IOException;
-
-    boolean getUserValidation() throws IOException;
-
 }

dspace-api/src/main/java/org/dspace/scripts/handler/impl/CommandLineDSpaceRunnableHandler.java

@@ -7,11 +7,9 @@
  */
 package org.dspace.scripts.handler.impl;
 
-import java.io.BufferedReader;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.InputStreamReader;
 
 import org.apache.commons.cli.HelpFormatter;
 import org.apache.commons.cli.Options;
@@ -108,14 +106,4 @@ public class CommandLineDSpaceRunnableHandler implements DSpaceRunnableHandler {
         File file = new File(fileName);
         FileUtils.copyInputStreamToFile(inputStream, file);
     }
-
-    @Override
-    public boolean getUserValidation() throws IOException {
-        logInfo("Do you want to make these changes? [y/n] ");
-        String yn = (new BufferedReader(new InputStreamReader(System.in))).readLine();
-        if ("y".equalsIgnoreCase(yn)) {
-            return true;
-        }
-        return false;
-    }
 }

dspace-api/src/main/java/org/dspace/scripts/service/ProcessService.java

@@ -135,8 +135,24 @@ public interface ProcessService {
      */
     public List<DSpaceCommandLineParameter> getParameters(Process process);
 
+    /**
+     * This method will return the Bitstream that matches the given name for the given Process
+     * @param context           The relevant DSpace context
+     * @param process           The process that should hold the requested Bitstream
+     * @param bitstreamName     The name of the requested Bitstream
+     * @return                  The Bitstream from the given Process that matches the given bitstream name
+     */
     public Bitstream getBitstreamByName(Context context, Process process, String bitstreamName);
 
+    /**
+     * This method will return all the Bitstreams for a given process if the type is defined as null. If type is
+     * different than null, the bitstreams with metadata process.type equal to the given type from that process
+     * are returned
+     * @param context   The relevant DSpace context
+     * @param process   The process that holds the Bitstreams to be searched in
+     * @param type      The type that the Bitstream must have
+     * @return          The list of Bitstreams of the given type for the given Process
+     */
     public List<Bitstream> getBitstreams(Context context, Process process, String type);
 
     /**

dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java

@@ -250,9 +250,4 @@ public class RestDSpaceRunnableHandler implements DSpaceRunnableHandler {
         }
         taskExecutor.execute(script);
     }
-
-    @Override
-    public boolean getUserValidation() throws IOException {
-        return true;
-    }
 }

dspace/config/spring/api/scripts.xml

@@ -9,7 +9,7 @@
         <property name="description" value="Update Discovery Solr Search Index"/>
     </bean>
 
-    <bean id="MetadataImport" class="org.dspace.app.bulkedit.MetadataImport" scope="prototype">
+    <bean id="MetadataImportCLI" class="org.dspace.app.bulkedit.MetadataImportCLI" scope="prototype">
         <property name="name" value="metadata-import" />
         <property name="description" value="Import metadata after batch editing" />
     </bean>

dspace/config/spring/rest/scripts.xml

@@ -6,4 +6,8 @@
         <property name="corePoolSize" value="5"/>
     </bean>
 
+    <bean id="MetadataImport" class="org.dspace.app.bulkedit.MetadataImport" scope="prototype">
+        <property name="name" value="metadata-import" />
+        <property name="description" value="Import metadata after batch editing" />
+    </bean>
 </beans>