[DS-204] New -zip option for item exporter and importer / [DS-216] Migrating items that use additional metadata schemas causes an NPE

git-svn-id: http://scm.dspace.org/svn/repo/dspace/trunk@3984 9c30dcfa-912a-0410-8fc2-9e0234be79fd

dspace-api/src/main/java/org/dspace/app/itemexport/ItemExport.java

@@ -136,6 +136,7 @@ public class ItemExport
         options.addOption("m", "migrate", false, "export for migration (remove handle and metadata that will be re-created in new system)");
         options.addOption("n", "number", true,
                 "sequence number to begin exporting items with");
+        options.addOption("z", "zip", true, "export as zip file (specify filename e.g. export.zip)");
         options.addOption("h", "help", false, "help");
 
         CommandLine line = parser.parse(options, argv);
@@ -196,6 +197,14 @@ public class ItemExport
             migrate = true;
         }
 
+        boolean zip = false;
+        String zipFileName = "";
+        if (line.hasOption('z'))
+        {
+            zip = true;
+            zipFileName = line.getOptionValue('z');
+        }
+
         // now validate the args
         if (myType == -1)
         {
@@ -279,25 +288,42 @@ public class ItemExport
             }
         }
 
-        if (myItem != null)
+        if (zip)
         {
-            // it's only a single item
+            ItemIterator items;
-            exportItem(c, myItem, destDirName, seqStart, migrate);
+            if (myItem != null)
+            {
+                items = new ItemIterator(c, new ArrayList(myItem.getID()));
+            }
+            else
+            {
+                System.out.println("Exporting from collection: " + myIDString);
+                items = mycollection.getItems();
+            }
+            exportAsZip(c, items, destDirName, zipFileName, seqStart, migrate);
         }
         else
         {
-            System.out.println("Exporting from collection: " + myIDString);
+            if (myItem != null)
-
-            // it's a collection, so do a bunch of items
-            ItemIterator i = mycollection.getItems();
-            try
             {
-                exportItem(c, i, destDirName, seqStart, migrate);
+                // it's only a single item
+                exportItem(c, myItem, destDirName, seqStart, migrate);
             }
-            finally
+            else
             {
-                if (i != null)
+                System.out.println("Exporting from collection: " + myIDString);
-                    i.close();
+
+                // it's a collection, so do a bunch of items
+                ItemIterator i = mycollection.getItems();
+                try
+                {
+                    exportItem(c, i, destDirName, seqStart, migrate);
+                }
+                finally
+                {
+                    if (i != null)
+                        i.close();
+                }
             }
         }
 
@@ -495,7 +521,10 @@ public class ItemExport
             }
 
             // When migrating, only keep date.issued if it is different to date.accessioned
-            if ((migrate) && (!dateIssued.equals(dateAccessioned)))
+            if ((migrate) &&
+                (dateIssued != null) &&
+                (dateAccessioned != null) &&
+                (!dateIssued.equals(dateAccessioned)))
             {
                 utf8 = ("  <dcvalue element=\"date\" "
                         + "qualifier=\"issued\">"
@@ -663,6 +692,44 @@ public class ItemExport
         }
     }
 
+    /**
+     * Method to perform an export and save it as a zip file.
+     *
+     * @param context The DSpace Context
+     * @param items The items to export
+     * @param destDirName The directory to save the export in
+     * @param zipFileName The name to save the zip file as
+     * @param seqStart The first number in the sequence
+     * @param migrate Whether to use the migrate option or not
+     * @throws Exception
+     */
+    public static void exportAsZip(Context context, ItemIterator items,
+                                   String destDirName, String zipFileName,
+                                   int seqStart, boolean migrate) throws Exception
+    {
+        String workDir = getExportWorkDirectory() +
+                         System.getProperty("file.separator") +
+                         zipFileName;
+
+        File wkDir = new File(workDir);
+        if (!wkDir.exists())
+        {
+            wkDir.mkdirs();
+        }
+
+        File dnDir = new File(destDirName);
+        if (!dnDir.exists())
+        {
+            dnDir.mkdirs();
+        }
+
+        // export the items using normal export method
+        exportItem(context, items, workDir, seqStart, migrate);
+
+        // now zip up the export directory created above
+        zip(workDir, destDirName + System.getProperty("file.separator") + zipFileName);
+    }
+
     /**
      * Convenience methot to create export a single Community, Collection, or
      * Item
@@ -1301,6 +1368,10 @@ public class ItemExport
             zipFiles(cpFile, strSource, tempFileName, cpZipOutputStream);
             cpZipOutputStream.finish();
             cpZipOutputStream.close();
+
+            // Fix issue on Windows with stale file handles open before trying to delete them
+            System.gc();
+
             deleteDirectory(cpFile);
             targetFile.renameTo(new File(target));
         }

dspace-api/src/main/java/org/dspace/app/itemimport/ItemImport.java

@@ -37,22 +37,11 @@
  */
 package org.dspace.app.itemimport;
 
-import java.io.BufferedInputStream;
+import java.io.*;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.io.PrintWriter;
 import java.sql.SQLException;
-import java.util.ArrayList;
+import java.util.*;
-import java.util.HashMap;
+import java.util.zip.ZipFile;
-import java.util.Iterator;
+import java.util.zip.ZipEntry;
-import java.util.Map;
-import java.util.StringTokenizer;
-import java.util.Vector;
 
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
@@ -151,6 +140,7 @@ public class ItemImport
         options.addOption("d", "delete", false,
                 "delete items listed in mapfile");
         options.addOption("s", "source", true, "source of items (directory)");
+        options.addOption("z", "zip", true, "name of zip file");
         options.addOption("c", "collection", true,
                 "destination collection(s) Handle or database ID");
         options.addOption("m", "mapfile", true, "mapfile items in mapfile");
@@ -180,7 +170,9 @@ public class ItemImport
             HelpFormatter myhelp = new HelpFormatter();
             myhelp.printHelp("ItemImport\n", options);
             System.out
-                    .println("\nadding items:    ItemImport -a -e eperson -c collection -s sourcedir -m mapfile");
+                   .println("\nadding items:    ItemImport -a -e eperson -c collection -s sourcedir -m mapfile");
+            System.out
+                    .println("\nadding items from zip file:    ItemImport -a -e eperson -c collection -s sourcedir -z filename.zip -m mapfile");
             System.out
                     .println("replacing items: ItemImport -r -e eperson -c collection -s sourcedir -m mapfile");
             System.out
@@ -249,6 +241,15 @@ public class ItemImport
                     .println("**Resume import** - attempting to import items not already imported");
         }
 
+        boolean zip = false;
+        String zipfilename = "";
+        String ziptempdir = ConfigurationManager.getProperty("org.dspace.app.itemexport.work.dir");
+        if (line.hasOption('z'))
+        {
+            zip = true;
+            zipfilename = sourcedir + System.getProperty("file.separator") + line.getOptionValue('z');
+        }
+
         // now validate
         // must have a command set
         if (command == null)
@@ -328,6 +329,36 @@ public class ItemImport
             System.exit(1);
         }
 
+        // does the zip file exist and can we write to the temp directory
+        if (zip)
+        {
+            File zipfile = new File(sourcedir);
+            if (!zipfile.canRead())
+            {
+                System.out.println("Zip file '" + sourcedir + "' does not exist, or is not readable.");
+                System.exit(1);
+            }
+
+            if (ziptempdir == null)
+            {
+                System.out.println("Unable to unzip import file as the key 'org.dspace.app.itemexport.work.dir' is not set in dspace.cfg");
+                System.exit(1);
+            }
+            zipfile = new File(ziptempdir);
+            if (!zipfile.isDirectory())
+            {
+                System.out.println("'" + ConfigurationManager.getProperty("org.dspace.app.itemexport.work.dir") +
+                                   "' as defined by the key 'org.dspace.app.itemexport.work.dir' in dspace.cfg " +
+                                   "is not a valid directory");
+                System.exit(1);
+            }
+            File tempdir = new File(ziptempdir);
+            tempdir.mkdirs();
+            sourcedir = ziptempdir + System.getProperty("file.separator") + line.getOptionValue("z");
+            ziptempdir = ziptempdir + System.getProperty("file.separator") +
+                         line.getOptionValue("z") + System.getProperty("file.separator");
+        }
+
         ItemImport myloader = new ItemImport();
 
         // create a context
@@ -412,6 +443,48 @@ public class ItemImport
 
         try
         {
+            // If this is a zip archive, unzip it first
+            if (zip)
+            {
+                ZipFile zf = new ZipFile(zipfilename);
+                ZipEntry entry;
+                Enumeration entries = zf.entries();
+                while (entries.hasMoreElements())
+                {
+                    entry = (ZipEntry)entries.nextElement();
+                    if (entry.isDirectory())
+                    {
+                        new File(ziptempdir + entry.getName()).mkdir();
+                    }
+                    else
+                    {
+                        System.out.println("Extracting file: " + entry.getName());
+                        int index = entry.getName().lastIndexOf('/');
+                        if (index == -1)
+                        {
+                            // Was it created on Windows instead?
+                            index = entry.getName().lastIndexOf('\\');
+                        }
+                        if (index > 0)
+                        {
+                            File dir = new File(ziptempdir + entry.getName().substring(0, index));
+                            dir.mkdirs();
+                        }
+                        byte[] buffer = new byte[1024];
+                        int len;
+                        InputStream in = zf.getInputStream(entry);
+                        BufferedOutputStream out = new BufferedOutputStream(
+                            new FileOutputStream(ziptempdir + entry.getName()));
+                        while((len = in.read(buffer)) >= 0)
+                        {
+                            out.write(buffer, 0, len);
+                        }
+                        in.close();
+                        out.close();
+                    }
+                }
+            }
+
             c.setIgnoreAuthorization(true);
 
             if (command.equals("add"))
@@ -446,6 +519,21 @@ public class ItemImport
             status = 1;
         }
 
+        // Delete the unzipped file
+        try
+        {
+            if (zip)
+            {
+                System.gc();
+                System.out.println("Deleting temporary zip directory: " + ziptempdir);
+                ItemImport.deleteDirectory(new File(ziptempdir));
+            }
+        }
+        catch (Exception ex)
+        {
+            System.out.println("Unable to delete temporary zip archive location: " + ziptempdir);
+        }
+
         if (mapOut != null)
         {
             mapOut.close();
@@ -1577,4 +1665,31 @@ public class ItemImport
 
         return builder.parse(new File(filename));
     }
+
+    /**
+     * Delete a directory and its child files and directories
+     * @param path The directory to delete
+     * @return Whether the deletion was successful or not
+     */
+    private static boolean deleteDirectory(File path)
+    {
+        if (path.exists())
+        {
+            File[] files = path.listFiles();
+            for (int i = 0; i < files.length; i++)
+            {
+                if (files[i].isDirectory())
+                {
+                    deleteDirectory(files[i]);
+                }
+                else
+                {
+                    files[i].delete();
+                }
+            }
+        }
+
+        boolean pathDeleted = path.delete();
+        return (pathDeleted);
+    }
 }

dspace/CHANGES

@@ -18,7 +18,9 @@
  - [DS-195] Allow the primary bitstream to be set in the item importer / exporter
  - [DS-196] METS exposed via OAI-PMH includes description.provenance information
  - [DS-200] SWORD module requires the X-Packaging header
+ - [DS-204] New -zip option for item exporter and importer
  - [DS-212] NPE thrown during Harvest of non-items when visibility restriction is enabled
+ - [DS-216] Migrating items that use additional metadata schemas causes an NPE
  - [DS-221] XMLUI 'current activity' recognises Google Chrome as Safari
 
 (Larry Stone)