Fix bulk creation via file upload

dspace-spring-rest/src/main/java/org/dspace/app/rest/RestResourceController.java

@@ -25,6 +25,7 @@ import javax.servlet.http.HttpServletResponse;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.log4j.Logger;
@@ -551,6 +552,8 @@ public class RestResourceController implements InitializingBean {
     public <T extends RestAddressableModel> ResponseEntity<ResourceSupport> upload(HttpServletRequest request,
                                                                                    @PathVariable String apiCategory,
                                                                                    @PathVariable String model,
+                                                                                   @RequestParam(required = false)
+                                                                                       String extraField,
                                                                                    @RequestParam("file") MultipartFile
                                                                                        uploadfile)
         throws SQLException, FileNotFoundException, IOException, AuthorizeException {
@@ -558,7 +561,7 @@ public class RestResourceController implements InitializingBean {
         checkModelPluralForm(apiCategory, model);
         DSpaceRestRepository repository = utils.getResourceRepository(apiCategory, model);
 
-        Iterable<T> content = repository.upload(request, uploadfile);
+        Iterable<T> content = repository.upload(request, extraField, uploadfile);
 
         List<DSpaceResource> resources = new ArrayList<>();
         for (T modelObject : content) {
@@ -630,7 +633,6 @@ public class RestResourceController implements InitializingBean {
                                                                                    String model, ID id,
                                                                                    JsonNode jsonNode)
         throws HttpRequestMethodNotSupportedException {
-
         checkModelPluralForm(apiCategory, model);
         DSpaceRestRepository<RestAddressableModel, ID> repository = utils.getResourceRepository(apiCategory, model);
         RestAddressableModel modelObject = null;

dspace-spring-rest/src/main/java/org/dspace/app/rest/repository/DSpaceRestRepository.java

@@ -373,6 +373,8 @@ public abstract class DSpaceRestRepository<T extends RestAddressableModel, ID ex
      * 
      * @param request
      *            the http request
+     * @param extraField
+     *            the original name of the uploaded file
      * @param uploadfile
      *            the file to process
      * @return the created objects
@@ -381,18 +383,21 @@ public abstract class DSpaceRestRepository<T extends RestAddressableModel, ID ex
      * @throws IOException
      * @throws AuthorizeException
      */
-    public Iterable<T> upload(HttpServletRequest request, MultipartFile uploadfile)
+    public Iterable<T> upload(HttpServletRequest request, String extraField, MultipartFile uploadfile)
         throws SQLException, FileNotFoundException, IOException, AuthorizeException {
         Context context = obtainContext();
-        Iterable<T> entity = upload(context, request, uploadfile);
+        Iterable<T> entity = upload(context, request, extraField, uploadfile);
         context.commit();
         return entity;
     }
 
     /**
      * Method to implement to support bulk creation of objects from a file
+     * 
      * @param request
      *            the http request
+     * @param extraField
+     *            the original name of the uploaded file
      * @param uploadfile
      *            the file to process
      * @return the created objects
@@ -402,7 +407,8 @@ public abstract class DSpaceRestRepository<T extends RestAddressableModel, ID ex
      * @throws AuthorizeException
      * @throws RepositoryMethodNotImplementedException
      */
-    protected Iterable<T> upload(Context context, HttpServletRequest request, MultipartFile uploadfile)
+    protected Iterable<T> upload(Context context, HttpServletRequest request, String extraField,
+            MultipartFile uploadfile)
         throws SQLException, FileNotFoundException, IOException, AuthorizeException {
         throw new RepositoryMethodNotImplementedException("No implementation found; Method not allowed!", "");
     }

dspace-spring-rest/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java

@@ -65,6 +65,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.PageImpl;
 import org.springframework.data.domain.Pageable;
+import org.springframework.data.rest.webmvc.json.patch.PatchException;
 import org.springframework.stereotype.Component;
 import org.springframework.web.multipart.MultipartFile;
 
@@ -325,6 +326,7 @@ public class WorkspaceItemRestRepository extends DSpaceRestRepository<WorkspaceI
 
                 } catch (Exception e) {
                     log.error(e.getMessage(), e);
+                    throw new PatchException("Error processing the patch request", e);
                 }
             }
         }
@@ -345,7 +347,8 @@ public class WorkspaceItemRestRepository extends DSpaceRestRepository<WorkspaceI
     }
 
     @Override
-    public Iterable<WorkspaceItemRest> upload(Context context, HttpServletRequest request, MultipartFile uploadfile)
+    public Iterable<WorkspaceItemRest> upload(Context context, HttpServletRequest request, String extraField,
+            MultipartFile uploadfile)
         throws SQLException, FileNotFoundException, IOException, AuthorizeException {
         File file = Utils.getFile(uploadfile, "upload-loader", "filedataloader");
         List<WorkspaceItemRest> results = new ArrayList<>();
@@ -360,7 +363,7 @@ public class WorkspaceItemRestRepository extends DSpaceRestRepository<WorkspaceI
             if (StringUtils.isNotBlank(uuid)) {
                 collection = collectionService.find(context, UUID.fromString(uuid));
             } else {
-                collection = collectionService.findAll(context, 1, 0).get(0);
+                collection = collectionService.findAuthorizedOptimized(context, Constants.ADD).get(0);
             }
 
             SubmissionConfig submissionConfig =
@@ -370,6 +373,13 @@ public class WorkspaceItemRestRepository extends DSpaceRestRepository<WorkspaceI
             List<ItemSubmissionLookupDTO> tmpResult = new ArrayList<ItemSubmissionLookupDTO>();
 
             TransformationEngine transformationEngine1 = submissionLookupService.getPhase1TransformationEngine();
+            TransformationSpec spec = new TransformationSpec();
+            // FIXME this is mostly due to the need to test. The BTE framework has an assert statement that check if the
+            // number of found record is less than the requested and treat 0 as is, instead, the implementation assume
+            // 0=unlimited this lead to test failure.
+            // It is unclear if BTE really respect values other than 0/MAX allowing us to put a protection against heavy
+            // load
+            spec.setNumberOfRecords(Integer.MAX_VALUE);
             if (transformationEngine1 != null) {
                 MultipleSubmissionLookupDataLoader dataLoader =
                     (MultipleSubmissionLookupDataLoader) transformationEngine1.getDataLoader();
@@ -383,7 +393,7 @@ public class WorkspaceItemRestRepository extends DSpaceRestRepository<WorkspaceI
                             (SubmissionLookupOutputGenerator) transformationEngine1.getOutputGenerator();
                         outputGenerator.setDtoList(new ArrayList<ItemSubmissionLookupDTO>());
                         log.debug("BTE transformation is about to start!");
-                        transformationEngine1.transform(new TransformationSpec());
+                        transformationEngine1.transform(spec);
                         log.debug("BTE transformation finished!");
                         tmpResult.addAll(outputGenerator.getDtoList());
                         if (!tmpResult.isEmpty()) {
@@ -417,7 +427,7 @@ public class WorkspaceItemRestRepository extends DSpaceRestRepository<WorkspaceI
                     outputGenerator.setDto(tmpResult.get(0));
 
                     try {
-                        transformationEngine2.transform(new TransformationSpec());
+                        transformationEngine2.transform(spec);
                         result = outputGenerator.getWitems();
                     } catch (BadTransformationSpec e1) {
                         e1.printStackTrace();
@@ -456,9 +466,8 @@ public class WorkspaceItemRestRepository extends DSpaceRestRepository<WorkspaceI
                                 Object stepInstance = stepClass.newInstance();
                                 if (UploadableStep.class.isAssignableFrom(stepClass)) {
                                     UploadableStep uploadableStep = (UploadableStep) stepInstance;
-                                    ErrorRest err = uploadableStep
+                                    ErrorRest err = uploadableStep.upload(context, submissionService, stepConfig, wi,
-                                        .upload(context, submissionService, stepConfig, wi, uploadfile,
+                                            uploadfile, extraField);
-                                            file.getAbsolutePath());
                                     if (err != null) {
                                         errors.add(err);
                                     }

dspace-spring-rest/src/main/java/org/dspace/app/rest/submit/SubmissionService.java

@@ -18,12 +18,14 @@ import org.apache.log4j.Logger;
 import org.atteo.evo.inflector.English;
 import org.dspace.app.rest.converter.BitstreamFormatConverter;
 import org.dspace.app.rest.converter.ResourcePolicyConverter;
+import org.dspace.app.rest.exception.RESTAuthorizationException;
 import org.dspace.app.rest.model.BitstreamRest;
 import org.dspace.app.rest.model.CheckSumRest;
 import org.dspace.app.rest.model.MetadataValueRest;
 import org.dspace.app.rest.model.ResourcePolicyRest;
 import org.dspace.app.rest.model.step.UploadBitstreamRest;
 import org.dspace.app.rest.utils.ContextUtil;
+import org.dspace.authorize.AuthorizeException;
 import org.dspace.authorize.ResourcePolicy;
 import org.dspace.content.Bitstream;
 import org.dspace.content.Collection;
@@ -31,12 +33,14 @@ import org.dspace.content.MetadataValue;
 import org.dspace.content.WorkspaceItem;
 import org.dspace.content.service.CollectionService;
 import org.dspace.content.service.WorkspaceItemService;
+import org.dspace.core.Constants;
 import org.dspace.core.Context;
 import org.dspace.core.Utils;
 import org.dspace.services.ConfigurationService;
 import org.dspace.services.RequestService;
 import org.dspace.services.model.Request;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.jdbc.datasource.init.UncategorizedScriptException;
 import org.springframework.stereotype.Component;
 
 /**
@@ -64,23 +68,35 @@ public class SubmissionService {
 
     public WorkspaceItem createWorkspaceItem(Context context, Request request) {
         WorkspaceItem wsi = null;
+        Collection collection = null;
         String collectionUUID = request.getHttpServletRequest().getParameter("collection");
+
         if (StringUtils.isBlank(collectionUUID)) {
-            String uuid = configurationService.getProperty("submission.default.collection");
+            collectionUUID = configurationService.getProperty("submission.default.collection");
-            Collection collection = null;
+        }
-            try {
+
-                if (StringUtils.isNotBlank(uuid)) {
+        try {
-                    collection = collectionService.find(context, UUID.fromString(uuid));
+            if (StringUtils.isNotBlank(collectionUUID)) {
+                collection = collectionService.find(context, UUID.fromString(collectionUUID));
+            } else {
+                final List<Collection> findAuthorizedOptimized = collectionService.findAuthorizedOptimized(context,
+                        Constants.ADD);
+                if (findAuthorizedOptimized != null && findAuthorizedOptimized.size() > 0) {
+                    collection = findAuthorizedOptimized.get(0);
                 } else {
-                    collection = collectionService.findAll(context, 1, 0).get(0);
+                    throw new RESTAuthorizationException("No collection suitable for submission for the current user");
                 }
-                wsi = workspaceItemService.create(context, collection, true);
-            } catch (Exception e) {
-                log.error(e.getMessage(), e);
             }
-        } else {
+
-            //TODO manage setup of default collection in the case WSI it is not null
+            if (collection == null) {
-            //TODO manage setup of collection discovered into request
+                throw new RESTAuthorizationException("collectionUUID=" + collectionUUID + " not found");
+            }
+            wsi = workspaceItemService.create(context, collection, true);
+        } catch (SQLException e) {
+            // wrap in a runtime exception as we cannot change the method signature
+            throw new UncategorizedScriptException(e.getMessage(), e);
+        } catch (AuthorizeException ae) {
+            throw new RESTAuthorizationException(ae);
         }
         return wsi;
     }

dspace-spring-rest/src/main/java/org/dspace/app/rest/submit/step/UploadStep.java

@@ -7,22 +7,32 @@
  */
 package org.dspace.app.rest.submit.step;
 
+import java.io.BufferedInputStream;
+import java.io.InputStream;
 import java.util.List;
 
+import org.apache.log4j.Logger;
+import org.dspace.app.rest.model.ErrorRest;
 import org.dspace.app.rest.model.patch.Operation;
 import org.dspace.app.rest.model.step.DataUpload;
 import org.dspace.app.rest.model.step.UploadBitstreamRest;
+import org.dspace.app.rest.repository.WorkspaceItemRestRepository;
 import org.dspace.app.rest.submit.AbstractRestProcessingStep;
 import org.dspace.app.rest.submit.SubmissionService;
+import org.dspace.app.rest.submit.UploadableStep;
 import org.dspace.app.rest.submit.factory.PatchOperationFactory;
 import org.dspace.app.rest.submit.factory.impl.PatchOperation;
 import org.dspace.app.util.SubmissionStepConfig;
 import org.dspace.content.Bitstream;
+import org.dspace.content.BitstreamFormat;
 import org.dspace.content.Bundle;
+import org.dspace.content.InProgressSubmission;
+import org.dspace.content.Item;
 import org.dspace.content.WorkspaceItem;
 import org.dspace.core.Constants;
 import org.dspace.core.Context;
 import org.dspace.services.model.Request;
+import org.springframework.web.multipart.MultipartFile;
 
 /**
  * Upload step for DSpace Spring Rest. Expose information about the bitstream
@@ -30,8 +40,10 @@ import org.dspace.services.model.Request;
  *
  * @author Luigi Andrea Pascarelli (luigiandrea.pascarelli at 4science.it)
  */
-public class UploadStep extends org.dspace.submit.step.UploadStep implements AbstractRestProcessingStep {
+public class UploadStep extends org.dspace.submit.step.UploadStep
+        implements AbstractRestProcessingStep, UploadableStep {
 
+    private static final Logger log = Logger.getLogger(UploadStep.class);
 
     @Override
     public DataUpload getData(SubmissionService submissionService, WorkspaceItem obj, SubmissionStepConfig config)
@@ -79,4 +91,54 @@ public class UploadStep extends org.dspace.submit.step.UploadStep implements Abs
 
     }
 
+    @Override
+    public ErrorRest upload(Context context, SubmissionService submissionService, SubmissionStepConfig stepConfig,
+                            InProgressSubmission wsi, MultipartFile file, String extraField) {
+
+        Bitstream source = null;
+        BitstreamFormat bf = null;
+
+        Item item = wsi.getItem();
+        List<Bundle> bundles = null;
+        try {
+            // do we already have a bundle?
+            bundles = itemService.getBundles(item, Constants.CONTENT_BUNDLE_NAME);
+
+            InputStream inputStream = new BufferedInputStream(file.getInputStream());
+            if (bundles.size() < 1) {
+                // set bundle's name to ORIGINAL
+                source = itemService.createSingleBitstream(context, inputStream, item, Constants.CONTENT_BUNDLE_NAME);
+            } else {
+                // we have a bundle already, just add bitstream
+                source = bitstreamService.create(context, bundles.get(0), inputStream);
+            }
+
+            source.setName(context, extraField);
+            source.setSource(context, file.getOriginalFilename());
+
+            // Identify the format
+            bf = bitstreamFormatService.guessFormat(context, source);
+            source.setFormat(context, bf);
+
+            // Update to DB
+            bitstreamService.update(context, source);
+            itemService.update(context, item);
+
+        } catch (Exception e) {
+            log.error(e.getMessage(), e);
+            ErrorRest result = new ErrorRest();
+            result.setMessage(e.getMessage());
+            if (bundles != null && bundles.size() > 0) {
+                result.getPaths().add(
+                    "/" + WorkspaceItemRestRepository.OPERATION_PATH_SECTIONS + "/" + stepConfig.getId() + "/files/" +
+                    bundles.get(0).getBitstreams().size());
+            } else {
+                result.getPaths()
+                    .add("/" + WorkspaceItemRestRepository.OPERATION_PATH_SECTIONS + "/" + stepConfig.getId());
+            }
+            return result;
+        }
+
+        return null;
+    }
 }