diff --git a/dspace-api/src/main/java/org/dspace/app/itemimport/ItemImport.java b/dspace-api/src/main/java/org/dspace/app/itemimport/ItemImport.java index 9f8bf4b75d..d9e2b46d34 100644 --- a/dspace-api/src/main/java/org/dspace/app/itemimport/ItemImport.java +++ b/dspace-api/src/main/java/org/dspace/app/itemimport/ItemImport.java @@ -16,10 +16,12 @@ import gr.ekt.bteio.generators.DSpaceOutputGenerator; import java.io.*; import java.sql.SQLException; +import java.text.SimpleDateFormat; import java.util.*; import java.util.zip.ZipFile; import java.util.zip.ZipEntry; +import javax.mail.MessagingException; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; @@ -30,9 +32,11 @@ import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Options; import org.apache.commons.cli.PosixParser; +import org.apache.commons.lang.RandomStringUtils; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.apache.xpath.XPathAPI; +import org.dspace.app.itemexport.ItemExportException; import org.dspace.authorize.AuthorizeException; import org.dspace.authorize.AuthorizeManager; import org.dspace.authorize.ResourcePolicy; @@ -40,15 +44,21 @@ import org.dspace.content.Bitstream; import org.dspace.content.BitstreamFormat; import org.dspace.content.Bundle; import org.dspace.content.Collection; +import org.dspace.content.Community; +import org.dspace.content.DSpaceObject; import org.dspace.content.FormatIdentifier; import org.dspace.content.InstallItem; import org.dspace.content.Item; +import org.dspace.content.ItemIterator; import org.dspace.content.MetadataField; import org.dspace.content.MetadataSchema; import org.dspace.content.WorkspaceItem; import org.dspace.core.ConfigurationManager; import org.dspace.core.Constants; import org.dspace.core.Context; +import org.dspace.core.Email; +import org.dspace.core.I18nUtil; +import org.dspace.core.LogManager; import org.dspace.eperson.EPerson; import org.dspace.eperson.Group; import org.dspace.handle.HandleManager; @@ -565,7 +575,7 @@ public class ItemImport } else if ("add-bte".equals(command)) { - myloader.addBTEItems(c, mycollections, sourcedir, mapfile, template, bteInputType); + myloader.addBTEItems(c, mycollections, sourcedir, mapfile, template, bteInputType, null); } // complete all transactions @@ -624,9 +634,39 @@ public class ItemImport System.exit(status); } + /** + * In this method, the BTE is instantiated. THe workflow generates the DSpace files + * necessary for the upload, and the default item import method is called + * @param c The contect + * @param mycollections The collections the items are inserted to + * @param sourceDir The filepath to the file to read data from + * @param mapFile The filepath to mapfile to be generated + * @param template + * @param inputType The type of the input data (bibtex, csv, etc.) + * @param workingDir The path to create temporary files (for command line or UI based) + * @throws Exception + */ private void addBTEItems(Context c, Collection[] mycollections, - String sourceDir, String mapFile, boolean template, String inputType) throws Exception + String sourceDir, String mapFile, boolean template, String inputType, String workingDir) throws Exception { + //Determine the folder where BTE will output the results + String outputFolder = null; + if (workingDir == null){ //This indicates a command line import, create a random path + File importDir = new File(ConfigurationManager.getProperty("org.dspace.app.batchitemimport.work.dir")); + if (!importDir.exists()){ + boolean success = importDir.mkdir(); + if (!success) { + log.info("Cannot create batch import directory!"); + throw new Exception("Cannot create batch import directory!"); + } + } + //Get a random folder in case two admins batch import data at the same time + outputFolder = importDir + File.separator + generateRandomFilename(true); + } + else { //This indicates a UI import, working dir is preconfigured + outputFolder = workingDir + File.separator + ".bte_output_dspace"; + } + TransformationEngine te = new DSpace().getSingletonService(TransformationEngine.class); DataLoaderService dls = new DSpace().getSingletonService(DataLoaderService.class); @@ -657,11 +697,10 @@ public class ItemImport te.setDataLoader(dataLoader); DSpaceOutputGenerator outputGenerator = new DSpaceOutputGenerator(outputMap); - outputGenerator.setOutputDirectory("./.bte_output_dspace"); + outputGenerator.setOutputDirectory(outputFolder); te.setOutputGenerator(outputGenerator); - try { TransformationResult res = te.transform(new TransformationSpec()); List output = res.getOutput(); @@ -672,10 +711,10 @@ public class ItemImport } ItemImport myloader = new ItemImport(); - myloader.addItems(c, mycollections, "./.bte_output_dspace", mapFile, template); + myloader.addItems(c, mycollections, outputFolder, mapFile, template); //remove files from output generator - deleteDirectory(new File("./.bte_output_dspace")); + deleteDirectory(new File(outputFolder)); } } @@ -1884,4 +1923,192 @@ public class ItemImport boolean pathDeleted = path.delete(); return (pathDeleted); } + + /** + * Generate a random filename based on current time + * @param hidden: add . as a prefix to make the file hidden + * @return the filename + */ + private static String generateRandomFilename(boolean hidden) + { + String filename = String.format("%s", RandomStringUtils.randomAlphanumeric(8)); + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd_HHmm"); + String datePart = sdf.format(new Date()); + filename = datePart+"_"+filename; + + return filename; + } + + /** + * Given an uploaded file, this method calls the method to instantiate a BTE instance to + * transform the input data and batch import them to DSpace + * @param file The input file to read data from + * @param collections The collections the created items will be inserted to + * @param bteInputType The input type of the data (bibtex, csv, etc.) + * @param context The context + * @throws Exception + */ + public static void processUploadableImport(File file, Collection[] collections, + String bteInputType, Context context) throws Exception + { + final EPerson eperson = context.getCurrentUser(); + final File myFile = file; + final Collection[] mycollections = collections; + final String myBteInputType = bteInputType; + + // if the file exists + if (file.exists()) + { + Thread go = new Thread() + { + public void run() + { + Context context = null; + ItemIterator iitems = null; + try + { + // create a new dspace context + context = new Context(); + context.setIgnoreAuthorization(true); + + File importDir = new File(ConfigurationManager.getProperty("org.dspace.app.batchitemimport.work.dir")); + if (!importDir.exists()){ + boolean success = importDir.mkdir(); + if (!success) { + log.info("Cannot create batch import directory!"); + throw new Exception(); + } + } + //Generate a random filename for the subdirectory of the specific import in case + //more that one batch imports take place at the same time + String subDirName = generateRandomFilename(false); + String workingDir = importDir.getAbsolutePath() + File.separator + subDirName; + + //Create the import working directory + boolean success = (new File(workingDir)).mkdir(); + if (!success) { + log.info("Cannot create batch import working directory!"); + throw new Exception(); + } + + //Create random mapfile; + String mapfile = workingDir + File.separator+ "mapfile"; + + ItemImport myloader = new ItemImport(); + myloader.addBTEItems(context, mycollections, myFile.getAbsolutePath(), mapfile, template, myBteInputType, workingDir); + + // email message letting user know the file is ready for + // download + emailSuccessMessage(context, eperson, mapfile); + + // return to enforcing auths + context.setIgnoreAuthorization(false); + } + catch (Exception e1) + { + try + { + emailErrorMessage(eperson, e1.getMessage()); + } + catch (Exception e) + { + // wont throw here + } + throw new IllegalStateException(e1); + } + finally + { + if (iitems != null) + { + iitems.close(); + } + + // close the mapfile writer + if (mapOut != null) + { + mapOut.close(); + } + + // Make sure the database connection gets closed in all conditions. + try { + context.complete(); + } catch (SQLException sqle) { + context.abort(); + } + } + } + + }; + + go.isDaemon(); + go.start(); + } + else { + log.error("Unable to find the uploadable file"); + } + } + + /** + * Since the BTE batch import is done in a new thread we are unable to communicate + * with calling method about success or failure. We accomplish this + * communication with email instead. Send a success email once the batch + * import is complete + * + * @param context + * - the current Context + * @param eperson + * - eperson to send the email to + * @param fileName + * - the filepath to the mapfile created by the batch import + * @throws MessagingException + */ + public static void emailSuccessMessage(Context context, EPerson eperson, + String fileName) throws MessagingException + { + try + { + Locale supportedLocale = I18nUtil.getEPersonLocale(eperson); + Email email = Email.getEmail(I18nUtil.getEmailFilename(supportedLocale, "bte_batch_import_success")); + email.addRecipient(eperson.getEmail()); + email.addArgument(fileName); + + email.send(); + } + catch (Exception e) + { + log.warn(LogManager.getHeader(context, "emailSuccessMessage", "cannot notify user of export"), e); + } + } + + /** + * Since the BTE batch import is done in a new thread we are unable to communicate + * with calling method about success or failure. We accomplis this + * communication with email instead. Send an error email if the batch + * import fails + * + * @param eperson + * - EPerson to send the error message to + * @param error + * - the error message + * @throws MessagingException + */ + public static void emailErrorMessage(EPerson eperson, String error) + throws MessagingException + { + log.warn("An error occured during item export, the user will be notified. " + error); + try + { + Locale supportedLocale = I18nUtil.getEPersonLocale(eperson); + Email email = Email.getEmail(I18nUtil.getEmailFilename(supportedLocale, "bte_batch_import_error")); + email.addRecipient(eperson.getEmail()); + email.addArgument(error); + email.addArgument(ConfigurationManager.getProperty("dspace.url") + "/feedback"); + + email.send(); + } + catch (Exception e) + { + log.warn("error during item export error notification", e); + } + } } \ No newline at end of file diff --git a/dspace-api/src/main/resources/Messages.properties b/dspace-api/src/main/resources/Messages.properties index aba764743d..6229c51daf 100644 --- a/dspace-api/src/main/resources/Messages.properties +++ b/dspace-api/src/main/resources/Messages.properties @@ -299,6 +299,12 @@ jsp.dspace-admin.group-eperson-select.title = Select EPerson jsp.dspace-admin.group-group-select.add = Add Group jsp.dspace-admin.group-group-select.heading = Select Group to Add to Group {0} jsp.dspace-admin.group-group-select.title = Select Group +jsp.dspace-admin.batchmetadataimport.title = Batch import metadata (BTE) +jsp.dspace-admin.batchmetadataimport.success = The batch import was successful +jsp.dspace-admin.batchmetadataimport.genericerror = An error occured! Please, try again! +jsp.dspace-admin.batchmetadataimport.selectfile = Select data file to upload +jsp.dspace-admin.batchmetadataimport.selectinputfile = Select the type of the input data +jsp.dspace-admin.batchmetadataimport.selectcollection = Select the collection the items will be imported to jsp.dspace-admin.metadataimport.title = Import metadata jsp.dspace-admin.metadataimport.apply = Apply changes jsp.dspace-admin.metadataimport.unknownerror = An unknown error has occurred @@ -590,6 +596,7 @@ jsp.layout.navbar-admin.items = Items jsp.layout.navbar-admin.logout = Log Out jsp.layout.navbar-admin.privateitems = Private Items jsp.layout.navbar-admin.metadataimport = Import metadata +jsp.layout.navbar-admin.batchmetadataimport = Batch import metadata (BTE) jsp.layout.navbar-admin.metadataregistry = Metadata
Registry jsp.layout.navbar-admin.statistics = Statistics jsp.layout.navbar-admin.supervisors = Supervisors diff --git a/dspace-jspui/src/main/java/org/dspace/app/webui/servlet/BatchMetadataImportServlet.java b/dspace-jspui/src/main/java/org/dspace/app/webui/servlet/BatchMetadataImportServlet.java new file mode 100644 index 0000000000..273733b9c8 --- /dev/null +++ b/dspace-jspui/src/main/java/org/dspace/app/webui/servlet/BatchMetadataImportServlet.java @@ -0,0 +1,175 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.webui.servlet; + +import java.io.*; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.ResourceBundle; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.commons.fileupload.FileUploadBase.FileSizeLimitExceededException; +import org.apache.log4j.Logger; +import org.dspace.app.webui.util.JSPManager; +import org.dspace.app.webui.util.FileUploadRequest; +import org.dspace.app.itemimport.DataLoaderService; +import org.dspace.app.itemimport.ItemImport; +import org.dspace.authorize.AuthorizeException; +import org.dspace.content.Collection; +import org.dspace.core.*; +import org.dspace.utils.DSpace; +import org.elasticsearch.common.collect.Lists; + +/** + * Servlet to batch import metadata via the BTE + * + * @author Stuart Lewis + */ +public class BatchMetadataImportServlet extends DSpaceServlet +{ + /** log4j category */ + private static Logger log = Logger.getLogger(BatchMetadataImportServlet.class); + + /** + * Respond to a post request for metadata bulk importing via csv + * + * @param context a DSpace Context object + * @param request the HTTP request + * @param response the HTTP response + * + * @throws ServletException + * @throws IOException + * @throws SQLException + * @throws AuthorizeException + */ + protected void doDSPost(Context context, HttpServletRequest request, + HttpServletResponse response) throws ServletException, IOException, + SQLException, AuthorizeException + { + // First, see if we have a multipart request (uploading a metadata file) + String contentType = request.getContentType(); + if ((contentType != null) && (contentType.indexOf("multipart/form-data") != -1)) + { + String message = null; + + // Process the file uploaded + try { + // Wrap multipart request to get the submission info + FileUploadRequest wrapper = new FileUploadRequest(request); + File f = wrapper.getFile("file"); + + int colId = Integer.parseInt(wrapper.getParameter("collection")); + Collection collection = Collection.find(context, colId); + + String inputType = wrapper.getParameter("inputType"); + + try { + ItemImport.processUploadableImport(f, new Collection[]{collection}, inputType, context); + + request.setAttribute("has-error", "false"); + + } catch (Exception e) { + request.setAttribute("has-error", "true"); + message = e.getMessage(); + e.printStackTrace(); + } + } catch (FileSizeLimitExceededException e) { + request.setAttribute("has-error", "true"); + message = e.getMessage(); + e.printStackTrace(); + } catch (Exception e) { + request.setAttribute("has-error", "true"); + message = e.getMessage(); + e.printStackTrace(); + } + + //Get all the possible data loaders from the Spring configuration + DataLoaderService dls = new DSpace().getSingletonService(DataLoaderService.class); + List inputTypes = Lists.newArrayList(dls.getDataLoaders().keySet()); + + request.setAttribute("input-types", inputTypes); + + //Get all collections + List collections = null; + String colIdS = request.getParameter("colId"); + if (colIdS!=null){ + collections = new ArrayList(); + collections.add(Collection.find(context, Integer.parseInt(colIdS))); + + } + else { + collections = Arrays.asList(Collection.findAll(context)); + } + + request.setAttribute("collections", collections); + + request.setAttribute("message", message); + + // Show the upload screen + JSPManager.showJSP(request, response, "/dspace-admin/batchmetadataimport.jsp"); + + } + else + { + request.setAttribute("has-error", "true"); + + // Show the upload screen + JSPManager.showJSP(request, response, "/dspace-admin/batchmetadataimport.jsp"); + } + } + + /** + * GET request is only ever used to show the upload form + * + * @param context + * a DSpace Context object + * @param request + * the HTTP request + * @param response + * the HTTP response + * + * @throws ServletException + * @throws IOException + * @throws SQLException + * @throws AuthorizeException + */ + protected void doDSGet(Context context, HttpServletRequest request, + HttpServletResponse response) throws ServletException, IOException, + SQLException, AuthorizeException + { + //Get all the possible data loaders from the Spring configuration + DataLoaderService dls = new DSpace().getSingletonService(DataLoaderService.class); + List inputTypes = Lists.newArrayList(dls.getDataLoaders().keySet()); + + request.setAttribute("input-types", inputTypes); + + //Get all collections + List collections = null; + String colIdS = request.getParameter("colId"); + if (colIdS!=null){ + collections = new ArrayList(); + collections.add(Collection.find(context, Integer.parseInt(colIdS))); + + } + else { + collections = Arrays.asList(Collection.findAll(context)); + } + + request.setAttribute("collections", collections); + + // Show the upload screen + JSPManager.showJSP(request, response, "/dspace-admin/batchmetadataimport.jsp"); + } + +} \ No newline at end of file diff --git a/dspace-jspui/src/main/webapp/WEB-INF/web.xml b/dspace-jspui/src/main/webapp/WEB-INF/web.xml index c936f00572..6c552f5c7e 100644 --- a/dspace-jspui/src/main/webapp/WEB-INF/web.xml +++ b/dspace-jspui/src/main/webapp/WEB-INF/web.xml @@ -297,6 +297,11 @@ org.dspace.app.webui.servlet.MetadataImportServlet + + batchmetadataimport + org.dspace.app.webui.servlet.BatchMetadataImportServlet + + metadata-field-registry org.dspace.app.webui.servlet.admin.MetadataFieldRegistryServlet @@ -618,6 +623,11 @@ /dspace-admin/metadataimport + + batchmetadataimport + /dspace-admin/batchmetadataimport + + metadata-field-registry /dspace-admin/metadata-field-registry diff --git a/dspace-jspui/src/main/webapp/dspace-admin/batchmetadataimport.jsp b/dspace-jspui/src/main/webapp/dspace-admin/batchmetadataimport.jsp new file mode 100644 index 0000000000..b2f990cc52 --- /dev/null +++ b/dspace-jspui/src/main/webapp/dspace-admin/batchmetadataimport.jsp @@ -0,0 +1,112 @@ +<%-- + + The contents of this file are subject to the license and copyright + detailed in the LICENSE and NOTICE files at the root of the source + tree and available online at + + http://www.dspace.org/license/ + +--%> +<%-- + - Form to upload a csv metadata file +--%> + +<%@ page contentType="text/html;charset=UTF-8" %> + +<%@ taglib uri="http://java.sun.com/jsp/jstl/fmt" prefix="fmt" %> + +<%@ taglib uri="http://www.dspace.org/dspace-tags.tld" prefix="dspace" %> + +<%@ page import="java.util.List" %> +<%@ page import="org.dspace.content.Collection" %> + +<% + + List inputTypes = (List)request.getAttribute("input-types"); + List collections = (List)request.getAttribute("collections"); + String hasErrorS = (String)request.getAttribute("has-error"); + boolean hasError = (hasErrorS==null) ? true : (Boolean.parseBoolean((String)request.getAttribute("has-error"))); + + String message = (String)request.getAttribute("message"); +%> + + + +

+ +<% + if (hasErrorS == null){ + + } + else if (hasError && message!=null){ +%> + <%= message %> +<% + } + else if (hasError && message==null){ +%> + +<% + } + else { +%> + +<% + } +%> + +
+ + +

+ +

+ +

+ +

+ +

+ +

+ +

+ +

+ +

+ +

+ +

+ +

+ +

+ " /> +

+ +
+ + \ No newline at end of file diff --git a/dspace-jspui/src/main/webapp/layout/navbar-admin.jsp b/dspace-jspui/src/main/webapp/layout/navbar-admin.jsp index 43ed88f296..363d41f318 100644 --- a/dspace-jspui/src/main/webapp/layout/navbar-admin.jsp +++ b/dspace-jspui/src/main/webapp/layout/navbar-admin.jsp @@ -154,6 +154,15 @@ + + + .gif" width="16" height="16"/> + + + + + + .gif" width="16" height="16"/> diff --git a/dspace/config/dspace.cfg b/dspace/config/dspace.cfg index 7366afa31c..689dde12a3 100644 --- a/dspace/config/dspace.cfg +++ b/dspace/config/dspace.cfg @@ -727,6 +727,9 @@ org.dspace.app.itemexport.life.span.hours = 48 # cummulative sizes are more than this entry the export is not kicked off org.dspace.app.itemexport.max.size = 200 +### Batch Item import settings ### +# The directory where the results of imports will be placed (mapfile, upload file) +org.dspace.app.batchitemimport.work.dir = ${dspace.dir}/imports # For backwards compatibility, the subscription emails by default include any modified items # uncomment the following entry for only new items to be emailed diff --git a/dspace/config/emails/bte_batch_import_error b/dspace/config/emails/bte_batch_import_error new file mode 100644 index 0000000000..11657f29df --- /dev/null +++ b/dspace/config/emails/bte_batch_import_error @@ -0,0 +1,19 @@ +# Email sent to DSpace users when they BTE batch import fails. +# +# Parameters: {0} the export error +# {1} the URL to the feedback page +# +# +# See org.dspace.core.Email for information on the format of this file. +# +Subject: DSpace - The batch import was not completed. +The batch import you initiated from the DSpace UI was not completed, due to the following reason: + {0} + +For more information you may contact your system administrator: + {1} + + + +The DSpace Team + diff --git a/dspace/config/emails/bte_batch_import_success b/dspace/config/emails/bte_batch_import_success new file mode 100644 index 0000000000..930ccd82ae --- /dev/null +++ b/dspace/config/emails/bte_batch_import_success @@ -0,0 +1,15 @@ +# Email sent to DSpace users when they successfully batch import items via BTE. +# +# Parameters: {0} the filepath to the mapfile created by the batch import +# +# +# See org.dspace.core.Email for information on the format of this file. +# +Subject: DSpace - Batch import successfully completed +The batch item import you initiated from the DSpace UI has completed successfully. + +You may find the mapfile for the import in the following path: {0} + + +The DSpace Team +