Merge pull request #8710 from 4Science/CST-6402

Add SAF import via remote URL

dspace-api/src/main/java/org/dspace/app/itemimport/ItemImport.java

@@ -11,6 +11,7 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URL;
 import java.nio.file.Files;
 import java.sql.SQLException;
 import java.util.ArrayList;
@@ -74,10 +75,11 @@ public class ItemImport extends DSpaceRunnable<ItemImportScriptConfiguration> {
     protected boolean isQuiet = false;
     protected boolean commandLineCollections = false;
     protected boolean zip = false;
+    protected boolean remoteUrl = false;
     protected String zipfilename = null;
     protected boolean help = false;
     protected File workDir = null;
-    private File workFile = null;
+    protected File workFile = null;
 
     protected static final CollectionService collectionService =
             ContentServiceFactory.getInstance().getCollectionService();
@@ -237,6 +239,9 @@ public class ItemImport extends DSpaceRunnable<ItemImportScriptConfiguration> {
             if (zip) {
                 FileUtils.deleteDirectory(new File(sourcedir));
                 FileUtils.deleteDirectory(workDir);
+                if (remoteUrl && workFile != null && workFile.exists()) {
+                    workFile.delete();
+                }
             }
 
             Date endTime = new Date();
@@ -253,6 +258,17 @@ public class ItemImport extends DSpaceRunnable<ItemImportScriptConfiguration> {
      * @param context
      */
     protected void validate(Context context) {
+        // check zip type: uploaded file or remote url
+        if (commandLine.hasOption('z')) {
+            zipfilename = commandLine.getOptionValue('z');
+        } else if (commandLine.hasOption('u')) {
+            remoteUrl = true;
+            zipfilename = commandLine.getOptionValue('u');
+        }
+        if (StringUtils.isBlank(zipfilename)) {
+            throw new UnsupportedOperationException("Must run with either name of zip file or url of zip file");
+        }
+
         if (command == null) {
             handler.logError("Must run with either add, replace, or remove (run with -h flag for details)");
             throw new UnsupportedOperationException("Must run with either add, replace, or remove");
@@ -295,7 +311,6 @@ public class ItemImport extends DSpaceRunnable<ItemImportScriptConfiguration> {
             handler.writeFilestream(context, MAPFILE_FILENAME, mapfileInputStream, MAPFILE_BITSTREAM_TYPE);
         } finally {
             mapFile.delete();
-            workFile.delete();
         }
     }
 
@@ -306,17 +321,24 @@ public class ItemImport extends DSpaceRunnable<ItemImportScriptConfiguration> {
      * @throws Exception
      */
     protected void readZip(Context context, ItemImportService itemImportService) throws Exception {
-        Optional<InputStream> optionalFileStream = handler.getFileStream(context, zipfilename);
+        Optional<InputStream> optionalFileStream = Optional.empty();
+        if (!remoteUrl) {
+            // manage zip via upload
+            optionalFileStream = handler.getFileStream(context, zipfilename);
+        } else {
+            // manage zip via remote url
+            optionalFileStream = Optional.ofNullable(new URL(zipfilename).openStream());
+        }
         if (optionalFileStream.isPresent()) {
             workFile = new File(itemImportService.getTempWorkDir() + File.separator
                     + zipfilename + "-" + context.getCurrentUser().getID());
             FileUtils.copyInputStreamToFile(optionalFileStream.get(), workFile);
-            workDir = new File(itemImportService.getTempWorkDir() + File.separator + TEMP_DIR);
-            sourcedir = itemImportService.unzip(workFile, workDir.getAbsolutePath());
         } else {
             throw new IllegalArgumentException(
                     "Error reading file, the file couldn't be found for filename: " + zipfilename);
         }
+        workDir = new File(itemImportService.getTempWorkDir() + File.separator + TEMP_DIR);
+        sourcedir = itemImportService.unzip(workFile, workDir.getAbsolutePath());
     }
 
     /**
@@ -356,7 +378,6 @@ public class ItemImport extends DSpaceRunnable<ItemImportScriptConfiguration> {
      */
     protected void setZip() {
         zip = true;
-        zipfilename = commandLine.getOptionValue('z');
     }
 
     /**

dspace-api/src/main/java/org/dspace/app/itemimport/ItemImportCLI.java

@@ -8,10 +8,14 @@
 package org.dspace.app.itemimport;
 
 import java.io.File;
+import java.io.InputStream;
+import java.net.URL;
 import java.sql.SQLException;
 import java.util.List;
+import java.util.Optional;
 import java.util.UUID;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.dspace.app.itemimport.service.ItemImportService;
 import org.dspace.content.Collection;
@@ -62,7 +66,7 @@ public class ItemImportCLI extends ItemImport {
             handler.logError("Must run with either add, replace, or remove (run with -h flag for details)");
             throw new UnsupportedOperationException("Must run with either add, replace, or remove");
         } else if ("add".equals(command) || "replace".equals(command)) {
-            if (sourcedir == null) {
+            if (!remoteUrl && sourcedir == null) {
                 handler.logError("A source directory containing items must be set (run with -h flag for details)");
                 throw new UnsupportedOperationException("A source directory containing items must be set");
             }
@@ -96,10 +100,25 @@ public class ItemImportCLI extends ItemImport {
     protected void readZip(Context context, ItemImportService itemImportService) throws Exception {
         // If this is a zip archive, unzip it first
         if (zip) {
+            if (!remoteUrl) {
                 workDir = new File(itemImportService.getTempWorkDir() + File.separator + TEMP_DIR
                         + File.separator + context.getCurrentUser().getID());
                 sourcedir = itemImportService.unzip(
                         new File(sourcedir + File.separator + zipfilename), workDir.getAbsolutePath());
+            } else {
+                // manage zip via remote url
+                Optional<InputStream> optionalFileStream = Optional.ofNullable(new URL(zipfilename).openStream());
+                if (optionalFileStream.isPresent()) {
+                    workFile = new File(itemImportService.getTempWorkDir() + File.separator
+                            + zipfilename + "-" + context.getCurrentUser().getID());
+                    FileUtils.copyInputStreamToFile(optionalFileStream.get(), workFile);
+                } else {
+                    throw new IllegalArgumentException(
+                            "Error reading file, the file couldn't be found for filename: " + zipfilename);
+                }
+                workDir = new File(itemImportService.getTempWorkDir() + File.separator + TEMP_DIR);
+                sourcedir = itemImportService.unzip(workFile, workDir.getAbsolutePath());
+            }
         }
     }
 
@@ -120,6 +139,12 @@ public class ItemImportCLI extends ItemImport {
             zip = true;
             zipfilename = commandLine.getOptionValue('z');
         }
+
+        if (commandLine.hasOption('u')) { // remote url
+            zip = true;
+            remoteUrl = true;
+            zipfilename = commandLine.getOptionValue('u');
+        }
     }
 
     @Override

dspace-api/src/main/java/org/dspace/app/itemimport/ItemImportCLIScriptConfiguration.java

@@ -37,6 +37,9 @@ public class ItemImportCLIScriptConfiguration extends ItemImportScriptConfigurat
         options.addOption(Option.builder("z").longOpt("zip")
                 .desc("name of zip file")
                 .hasArg().required(false).build());
+        options.addOption(Option.builder("u").longOpt("url")
+                .desc("url of zip file")
+                .hasArg().build());
         options.addOption(Option.builder("c").longOpt("collection")
                 .desc("destination collection(s) Handle or database ID")
                 .hasArg().required(false).build());

dspace-api/src/main/java/org/dspace/app/itemimport/ItemImportScriptConfiguration.java

@@ -64,7 +64,10 @@ public class ItemImportScriptConfiguration<T extends ItemImport> extends ScriptC
         options.addOption(Option.builder("z").longOpt("zip")
                 .desc("name of zip file")
                 .type(InputStream.class)
-                .hasArg().required().build());
+                .hasArg().build());
+        options.addOption(Option.builder("u").longOpt("url")
+                .desc("url of zip file")
+                .hasArg().build());
         options.addOption(Option.builder("c").longOpt("collection")
                 .desc("destination collection(s) Handle or database ID")
                 .hasArg().required(false).build());