hazza/DSpace
1
0
Fork 0
mirror of https://github.com/DSpace/DSpace.git synced 2025-10-07 18:14:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix bulk creation via file upload

Browse Source
This commit is contained in:
Andrea Bollini
2018-10-07 22:20:45 +02:00
parent 1bb1fd44a4
commit 951d039830
5 changed files with 120 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
dspace-spring-rest/src/main/java/org/dspace/app/rest/RestResourceController.java
View File

@@ -25,6 +25,7 @@ import javax.servlet.http.HttpServletResponse;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
@@ -551,6 +552,8 @@ public class RestResourceController implements InitializingBean {
public <T extends RestAddressableModel> ResponseEntity<ResourceSupport> upload(HttpServletRequest request,
@PathVariable String apiCategory,
@PathVariable String model,
@RequestParam(required = false)
String extraField,
@RequestParam("file") MultipartFile
uploadfile)
throws SQLException, FileNotFoundException, IOException, AuthorizeException {
@@ -558,7 +561,7 @@ public class RestResourceController implements InitializingBean {
checkModelPluralForm(apiCategory, model);
DSpaceRestRepository repository = utils.getResourceRepository(apiCategory, model);
Iterable<T> content = repository.upload(request, uploadfile);
Iterable<T> content = repository.upload(request, extraField, uploadfile);
List<DSpaceResource> resources = new ArrayList<>();
for (T modelObject : content) {
@@ -630,7 +633,6 @@ public class RestResourceController implements InitializingBean {
String model, ID id,
JsonNode jsonNode)
throws HttpRequestMethodNotSupportedException {
checkModelPluralForm(apiCategory, model);
DSpaceRestRepository<RestAddressableModel, ID> repository = utils.getResourceRepository(apiCategory, model);
RestAddressableModel modelObject = null;

12
dspace-spring-rest/src/main/java/org/dspace/app/rest/repository/DSpaceRestRepository.java
View File

@@ -373,6 +373,8 @@ public abstract class DSpaceRestRepository<T extends RestAddressableModel, ID ex
*
* @param request
* the http request
* @param extraField
* the original name of the uploaded file
* @param uploadfile
* the file to process
* @return the created objects
@@ -381,18 +383,21 @@ public abstract class DSpaceRestRepository<T extends RestAddressableModel, ID ex
* @throws IOException
* @throws AuthorizeException
*/
public Iterable<T> upload(HttpServletRequest request, MultipartFile uploadfile)
public Iterable<T> upload(HttpServletRequest request, String extraField, MultipartFile uploadfile)
throws SQLException, FileNotFoundException, IOException, AuthorizeException {
Context context = obtainContext();
Iterable<T> entity = upload(context, request, uploadfile);
Iterable<T> entity = upload(context, request, extraField, uploadfile);
context.commit();
return entity;
}
/**
* Method to implement to support bulk creation of objects from a file
*
* @param request
* the http request
* @param extraField
* the original name of the uploaded file
* @param uploadfile
* the file to process
* @return the created objects
@@ -402,7 +407,8 @@ public abstract class DSpaceRestRepository<T extends RestAddressableModel, ID ex
* @throws AuthorizeException
* @throws RepositoryMethodNotImplementedException
*/
protected Iterable<T> upload(Context context, HttpServletRequest request, MultipartFile uploadfile)
protected Iterable<T> upload(Context context, HttpServletRequest request, String extraField,
MultipartFile uploadfile)
throws SQLException, FileNotFoundException, IOException, AuthorizeException {
throw new RepositoryMethodNotImplementedException("No implementation found; Method not allowed!", "");
}

23
dspace-spring-rest/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java
View File

@@ -65,6 +65,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.Pageable;
import org.springframework.data.rest.webmvc.json.patch.PatchException;
import org.springframework.stereotype.Component;
import org.springframework.web.multipart.MultipartFile;
@@ -325,6 +326,7 @@ public class WorkspaceItemRestRepository extends DSpaceRestRepository<WorkspaceI
} catch (Exception e) {
log.error(e.getMessage(), e);
throw new PatchException("Error processing the patch request", e);
}
}
}
@@ -345,7 +347,8 @@ public class WorkspaceItemRestRepository extends DSpaceRestRepository<WorkspaceI
}
@Override
public Iterable<WorkspaceItemRest> upload(Context context, HttpServletRequest request, MultipartFile uploadfile)
public Iterable<WorkspaceItemRest> upload(Context context, HttpServletRequest request, String extraField,
MultipartFile uploadfile)
throws SQLException, FileNotFoundException, IOException, AuthorizeException {
File file = Utils.getFile(uploadfile, "upload-loader", "filedataloader");
List<WorkspaceItemRest> results = new ArrayList<>();
@@ -360,7 +363,7 @@ public class WorkspaceItemRestRepository extends DSpaceRestRepository<WorkspaceI
if (StringUtils.isNotBlank(uuid)) {
collection = collectionService.find(context, UUID.fromString(uuid));
} else {
collection = collectionService.findAll(context, 1, 0).get(0);
collection = collectionService.findAuthorizedOptimized(context, Constants.ADD).get(0);
}
SubmissionConfig submissionConfig =
@@ -370,6 +373,13 @@ public class WorkspaceItemRestRepository extends DSpaceRestRepository<WorkspaceI
List<ItemSubmissionLookupDTO> tmpResult = new ArrayList<ItemSubmissionLookupDTO>();
TransformationEngine transformationEngine1 = submissionLookupService.getPhase1TransformationEngine();
TransformationSpec spec = new TransformationSpec();
// FIXME this is mostly due to the need to test. The BTE framework has an assert statement that check if the
// number of found record is less than the requested and treat 0 as is, instead, the implementation assume
// 0=unlimited this lead to test failure.
// It is unclear if BTE really respect values other than 0/MAX allowing us to put a protection against heavy
// load
spec.setNumberOfRecords(Integer.MAX_VALUE);
if (transformationEngine1 != null) {
MultipleSubmissionLookupDataLoader dataLoader =
(MultipleSubmissionLookupDataLoader) transformationEngine1.getDataLoader();
@@ -383,7 +393,7 @@ public class WorkspaceItemRestRepository extends DSpaceRestRepository<WorkspaceI
(SubmissionLookupOutputGenerator) transformationEngine1.getOutputGenerator();
outputGenerator.setDtoList(new ArrayList<ItemSubmissionLookupDTO>());
log.debug("BTE transformation is about to start!");
transformationEngine1.transform(new TransformationSpec());
transformationEngine1.transform(spec);
log.debug("BTE transformation finished!");
tmpResult.addAll(outputGenerator.getDtoList());
if (!tmpResult.isEmpty()) {
@@ -417,7 +427,7 @@ public class WorkspaceItemRestRepository extends DSpaceRestRepository<WorkspaceI
outputGenerator.setDto(tmpResult.get(0));
try {
transformationEngine2.transform(new TransformationSpec());
transformationEngine2.transform(spec);
result = outputGenerator.getWitems();
} catch (BadTransformationSpec e1) {
e1.printStackTrace();
@@ -456,9 +466,8 @@ public class WorkspaceItemRestRepository extends DSpaceRestRepository<WorkspaceI
Object stepInstance = stepClass.newInstance();
if (UploadableStep.class.isAssignableFrom(stepClass)) {
UploadableStep uploadableStep = (UploadableStep) stepInstance;
ErrorRest err = uploadableStep
.upload(context, submissionService, stepConfig, wi, uploadfile,
file.getAbsolutePath());
ErrorRest err = uploadableStep.upload(context, submissionService, stepConfig, wi,
uploadfile, extraField);
if (err != null) {
errors.add(err);
}

40
dspace-spring-rest/src/main/java/org/dspace/app/rest/submit/SubmissionService.java
View File

@@ -18,12 +18,14 @@ import org.apache.log4j.Logger;
import org.atteo.evo.inflector.English;
import org.dspace.app.rest.converter.BitstreamFormatConverter;
import org.dspace.app.rest.converter.ResourcePolicyConverter;
import org.dspace.app.rest.exception.RESTAuthorizationException;
import org.dspace.app.rest.model.BitstreamRest;
import org.dspace.app.rest.model.CheckSumRest;
import org.dspace.app.rest.model.MetadataValueRest;
import org.dspace.app.rest.model.ResourcePolicyRest;
import org.dspace.app.rest.model.step.UploadBitstreamRest;
import org.dspace.app.rest.utils.ContextUtil;
import org.dspace.authorize.AuthorizeException;
import org.dspace.authorize.ResourcePolicy;
import org.dspace.content.Bitstream;
import org.dspace.content.Collection;
@@ -31,12 +33,14 @@ import org.dspace.content.MetadataValue;
import org.dspace.content.WorkspaceItem;
import org.dspace.content.service.CollectionService;
import org.dspace.content.service.WorkspaceItemService;
import org.dspace.core.Constants;
import org.dspace.core.Context;
import org.dspace.core.Utils;
import org.dspace.services.ConfigurationService;
import org.dspace.services.RequestService;
import org.dspace.services.model.Request;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.datasource.init.UncategorizedScriptException;
import org.springframework.stereotype.Component;
/**
@@ -64,23 +68,35 @@ public class SubmissionService {
public WorkspaceItem createWorkspaceItem(Context context, Request request) {
WorkspaceItem wsi = null;
Collection collection = null;
String collectionUUID = request.getHttpServletRequest().getParameter("collection");
if (StringUtils.isBlank(collectionUUID)) {
String uuid = configurationService.getProperty("submission.default.collection");
Collection collection = null;
try {
if (StringUtils.isNotBlank(uuid)) {
collection = collectionService.find(context, UUID.fromString(uuid));
collectionUUID = configurationService.getProperty("submission.default.collection");
}
try {
if (StringUtils.isNotBlank(collectionUUID)) {
collection = collectionService.find(context, UUID.fromString(collectionUUID));
} else {
final List<Collection> findAuthorizedOptimized = collectionService.findAuthorizedOptimized(context,
Constants.ADD);
if (findAuthorizedOptimized != null && findAuthorizedOptimized.size() > 0) {
collection = findAuthorizedOptimized.get(0);
} else {
collection = collectionService.findAll(context, 1, 0).get(0);
throw new RESTAuthorizationException("No collection suitable for submission for the current user");
}
wsi = workspaceItemService.create(context, collection, true);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
} else {
//TODO manage setup of default collection in the case WSI it is not null
//TODO manage setup of collection discovered into request
if (collection == null) {
throw new RESTAuthorizationException("collectionUUID=" + collectionUUID + " not found");
}
wsi = workspaceItemService.create(context, collection, true);
} catch (SQLException e) {
// wrap in a runtime exception as we cannot change the method signature
throw new UncategorizedScriptException(e.getMessage(), e);
} catch (AuthorizeException ae) {
throw new RESTAuthorizationException(ae);
}
return wsi;
}

64
dspace-spring-rest/src/main/java/org/dspace/app/rest/submit/step/UploadStep.java
View File

@@ -7,22 +7,32 @@
*/
package org.dspace.app.rest.submit.step;
import java.io.BufferedInputStream;
import java.io.InputStream;
import java.util.List;
import org.apache.log4j.Logger;
import org.dspace.app.rest.model.ErrorRest;
import org.dspace.app.rest.model.patch.Operation;
import org.dspace.app.rest.model.step.DataUpload;
import org.dspace.app.rest.model.step.UploadBitstreamRest;
import org.dspace.app.rest.repository.WorkspaceItemRestRepository;
import org.dspace.app.rest.submit.AbstractRestProcessingStep;
import org.dspace.app.rest.submit.SubmissionService;
import org.dspace.app.rest.submit.UploadableStep;
import org.dspace.app.rest.submit.factory.PatchOperationFactory;
import org.dspace.app.rest.submit.factory.impl.PatchOperation;
import org.dspace.app.util.SubmissionStepConfig;
import org.dspace.content.Bitstream;
import org.dspace.content.BitstreamFormat;
import org.dspace.content.Bundle;
import org.dspace.content.InProgressSubmission;
import org.dspace.content.Item;
import org.dspace.content.WorkspaceItem;
import org.dspace.core.Constants;
import org.dspace.core.Context;
import org.dspace.services.model.Request;
import org.springframework.web.multipart.MultipartFile;
/**
* Upload step for DSpace Spring Rest. Expose information about the bitstream
@@ -30,8 +40,10 @@ import org.dspace.services.model.Request;
*
* @author Luigi Andrea Pascarelli (luigiandrea.pascarelli at 4science.it)
*/
public class UploadStep extends org.dspace.submit.step.UploadStep implements AbstractRestProcessingStep {
public class UploadStep extends org.dspace.submit.step.UploadStep
implements AbstractRestProcessingStep, UploadableStep {
private static final Logger log = Logger.getLogger(UploadStep.class);
@Override
public DataUpload getData(SubmissionService submissionService, WorkspaceItem obj, SubmissionStepConfig config)
@@ -79,4 +91,54 @@ public class UploadStep extends org.dspace.submit.step.UploadStep implements Abs
}
@Override
public ErrorRest upload(Context context, SubmissionService submissionService, SubmissionStepConfig stepConfig,
InProgressSubmission wsi, MultipartFile file, String extraField) {
Bitstream source = null;
BitstreamFormat bf = null;
Item item = wsi.getItem();
List<Bundle> bundles = null;
try {
// do we already have a bundle?
bundles = itemService.getBundles(item, Constants.CONTENT_BUNDLE_NAME);
InputStream inputStream = new BufferedInputStream(file.getInputStream());
if (bundles.size() < 1) {
// set bundle's name to ORIGINAL
source = itemService.createSingleBitstream(context, inputStream, item, Constants.CONTENT_BUNDLE_NAME);
} else {
// we have a bundle already, just add bitstream
source = bitstreamService.create(context, bundles.get(0), inputStream);
}
source.setName(context, extraField);
source.setSource(context, file.getOriginalFilename());
// Identify the format
bf = bitstreamFormatService.guessFormat(context, source);
source.setFormat(context, bf);
// Update to DB
bitstreamService.update(context, source);
itemService.update(context, item);
} catch (Exception e) {
log.error(e.getMessage(), e);
ErrorRest result = new ErrorRest();
result.setMessage(e.getMessage());
if (bundles != null && bundles.size() > 0) {
result.getPaths().add(
"/" + WorkspaceItemRestRepository.OPERATION_PATH_SECTIONS + "/" + stepConfig.getId() + "/files/" +
bundles.get(0).getBitstreams().size());
} else {
result.getPaths()
.add("/" + WorkspaceItemRestRepository.OPERATION_PATH_SECTIONS + "/" + stepConfig.getId());
}
return result;
}
return null;
}
}