hazza/DSpace
1
0
Fork 0
mirror of https://github.com/DSpace/DSpace.git synced 2025-10-07 01:54:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implementing file import and export scripts

Browse Source
This commit is contained in:
Raf Ponsaerts
2019-11-21 13:22:09 +01:00
parent 0ea677914f
commit 08c7da4446
15 changed files with 1501 additions and 1036 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
dspace-api/src/main/java/org/dspace/app/bulkedit/DSpaceCSV.java
View File

@@ -13,6 +13,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Serializable;
@@ -27,7 +28,9 @@ import java.util.UUID;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.pdfbox.util.Charsets;
import org.dspace.authority.AuthorityValue;
import org.dspace.authority.factory.AuthorityServiceFactory;
import org.dspace.authority.service.AuthorityValueService;
@@ -637,6 +640,14 @@ public class DSpaceCSV implements Serializable {
out.close();
}
public InputStream getInputStream() {
StringBuilder stringBuilder = new StringBuilder();
for (String csvLine : getCSVLinesAsStringArray()) {
stringBuilder.append(csvLine + "\n");
}
return IOUtils.toInputStream(stringBuilder.toString(), Charsets.UTF_8);
}
/**
* Is it Ok to export this value? When exportAll is set to false, we don't export
* some of the metadata elements.

288
dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExport.java
View File

@@ -7,272 +7,84 @@
*/
package org.dspace.app.bulkedit;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import com.google.common.collect.Iterators;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
import org.dspace.content.Collection;
import org.dspace.content.Community;
import org.dspace.content.DSpaceObject;
import org.dspace.content.Item;
import org.dspace.content.factory.ContentServiceFactory;
import org.dspace.content.service.ItemService;
import org.dspace.core.Constants;
import org.dspace.app.bulkedit.DSpaceCSV;
import org.dspace.content.service.MetadataExportService;
import org.dspace.core.Context;
import org.dspace.handle.factory.HandleServiceFactory;
import org.dspace.scripts.DSpaceRunnable;
import org.springframework.beans.factory.annotation.Autowired;
/**
* Metadata exporter to allow the batch export of metadata into a file
*
* @author Stuart Lewis
*/
public class MetadataExport {
/**
* The items to export
*/
protected Iterator<Item> toExport;
public class MetadataExport extends DSpaceRunnable {
protected ItemService itemService;
private Context c = null;
private boolean help = false;
private String filename = null;
private String handle = null;
private boolean exportAllMetadata = false;
private boolean exportAllItems = false;
@Autowired
private MetadataExportService metadataExportService;
protected Context context;
/**
* Whether to export all metadata, or just normally edited metadata
*/
protected boolean exportAll;
protected MetadataExport() {
itemService = ContentServiceFactory.getInstance().getItemService();
private MetadataExport() {
Options options = constructOptions();
this.options = options;
}
/**
* Set up a new metadata export
*
* @param c The Context
* @param toExport The ItemIterator of items to export
* @param exportAll whether to export all metadata or not (include handle, provenance etc)
*/
public MetadataExport(Context c, Iterator<Item> toExport, boolean exportAll) {
itemService = ContentServiceFactory.getInstance().getItemService();
// Store the export settings
this.toExport = toExport;
this.exportAll = exportAll;
this.context = c;
}
/**
* Method to export a community (and sub-communities and collections)
*
* @param c The Context
* @param toExport The Community to export
* @param exportAll whether to export all metadata or not (include handle, provenance etc)
*/
public MetadataExport(Context c, Community toExport, boolean exportAll) {
itemService = ContentServiceFactory.getInstance().getItemService();
try {
// Try to export the community
this.toExport = buildFromCommunity(c, toExport, 0);
this.exportAll = exportAll;
this.context = c;
} catch (SQLException sqle) {
// Something went wrong...
System.err.println("Error running exporter:");
sqle.printStackTrace(System.err);
System.exit(1);
}
}
/**
* Build an array list of item ids that are in a community (include sub-communities and collections)
*
* @param context DSpace context
* @param community The community to build from
* @param indent How many spaces to use when writing out the names of items added
* @return The list of item ids
* @throws SQLException if database error
*/
protected Iterator<Item> buildFromCommunity(Context context, Community community, int indent)
throws SQLException {
// Add all the collections
List<Collection> collections = community.getCollections();
Iterator<Item> result = null;
for (Collection collection : collections) {
for (int i = 0; i < indent; i++) {
System.out.print(" ");
}
Iterator<Item> items = itemService.findByCollection(context, collection);
result = addItemsToResult(result, items);
}
// Add all the sub-communities
List<Community> communities = community.getSubcommunities();
for (Community subCommunity : communities) {
for (int i = 0; i < indent; i++) {
System.out.print(" ");
}
Iterator<Item> items = buildFromCommunity(context, subCommunity, indent + 1);
result = addItemsToResult(result, items);
}
return result;
}
private Iterator<Item> addItemsToResult(Iterator<Item> result, Iterator<Item> items) {
if (result == null) {
result = items;
} else {
result = Iterators.concat(result, items);
}
return result;
}
/**
* Run the export
*
* @return the exported CSV lines
*/
public DSpaceCSV export() {
try {
Context.Mode originalMode = context.getCurrentMode();
context.setMode(Context.Mode.READ_ONLY);
// Process each item
DSpaceCSV csv = new DSpaceCSV(exportAll);
while (toExport.hasNext()) {
Item item = toExport.next();
csv.addItem(item);
context.uncacheEntity(item);
}
context.setMode(originalMode);
// Return the results
return csv;
} catch (Exception e) {
// Something went wrong...
System.err.println("Error exporting to CSV:");
e.printStackTrace();
return null;
}
}
/**
* Print the help message
*
* @param options The command line options the user gave
* @param exitCode the system exit code to use
*/
private static void printHelp(Options options, int exitCode) {
// print the help message
HelpFormatter myhelp = new HelpFormatter();
myhelp.printHelp("MetadataExport\n", options);
System.out.println("\nfull export: metadataexport -f filename");
System.out.println("partial export: metadataexport -i handle -f filename");
System.exit(exitCode);
}
/**
* main method to run the metadata exporter
*
* @param argv the command line arguments given
* @throws Exception if error occurs
*/
public static void main(String[] argv) throws Exception {
// Create an options object and populate it
CommandLineParser parser = new PosixParser();
private Options constructOptions() {
Options options = new Options();
options.addOption("i", "id", true, "ID or handle of thing to export (item, collection, or community)");
options.addOption("f", "file", true, "destination where you want file written");
options.getOption("f").setRequired(true);
options.addOption("a", "all", false,
"include all metadata fields that are not normally changed (e.g. provenance)");
options.addOption("h", "help", false, "help");
CommandLine line = null;
try {
line = parser.parse(options, argv);
} catch (ParseException pe) {
System.err.println("Error with commands.");
printHelp(options, 1);
System.exit(0);
return options;
}
if (line.hasOption('h')) {
printHelp(options, 0);
public void internalRun() throws Exception {
if (help) {
handler.logInfo("\nfull export: metadataexport -f filename");
handler.logInfo("partial export: metadataexport -i handle -f filename");
printHelp();
return;
}
// Check a filename is given
if (!line.hasOption('f')) {
System.err.println("Required parameter -f missing!");
printHelp(options, 1);
}
String filename = line.getOptionValue('f');
// Create a context
Context c = new Context(Context.Mode.READ_ONLY);
c.turnOffAuthorisationSystem();
// The things we'll export
Iterator<Item> toExport = null;
MetadataExport exporter = null;
// Export everything?
boolean exportAll = line.hasOption('a');
ContentServiceFactory contentServiceFactory = ContentServiceFactory.getInstance();
// Check we have an item OK
ItemService itemService = contentServiceFactory.getItemService();
if (!line.hasOption('i')) {
System.out.println("Exporting whole repository WARNING: May take some time!");
exporter = new MetadataExport(c, itemService.findAll(c), exportAll);
} else {
String handle = line.getOptionValue('i');
DSpaceObject dso = HandleServiceFactory.getInstance().getHandleService().resolveToObject(c, handle);
if (dso == null) {
System.err.println("Item '" + handle + "' does not resolve to an item in your repository!");
printHelp(options, 1);
}
if (dso.getType() == Constants.ITEM) {
System.out.println("Exporting item '" + dso.getName() + "' (" + handle + ")");
List<Item> item = new ArrayList<>();
item.add((Item) dso);
exporter = new MetadataExport(c, item.iterator(), exportAll);
} else if (dso.getType() == Constants.COLLECTION) {
System.out.println("Exporting collection '" + dso.getName() + "' (" + handle + ")");
Collection collection = (Collection) dso;
toExport = itemService.findByCollection(c, collection);
exporter = new MetadataExport(c, toExport, exportAll);
} else if (dso.getType() == Constants.COMMUNITY) {
System.out.println("Exporting community '" + dso.getName() + "' (" + handle + ")");
exporter = new MetadataExport(c, (Community) dso, exportAll);
} else {
System.err.println("Error identifying '" + handle + "'");
System.exit(1);
}
}
// Perform the export
DSpaceCSV csv = exporter.export();
// Save the files to the file
csv.save(filename);
// Finish off and tidy up
DSpaceCSV dSpaceCSV = metadataExportService.handleExport(c, exportAllItems, exportAllMetadata, handle);
handler.writeFilestream(c, filename, dSpaceCSV.getInputStream(), "exportCSV");
c.restoreAuthSystemState();
c.complete();
}
public void setup() throws ParseException {
c = new Context();
c.turnOffAuthorisationSystem();
if (commandLine.hasOption('h')) {
help = true;
}
// Check a filename is given
if (!commandLine.hasOption('f')) {
throw new ParseException("Required parameter -f missing!");
}
filename = commandLine.getOptionValue('f');
exportAllMetadata = commandLine.hasOption('a');
if (commandLine.hasOption('i')) {
exportAllItems = true;
}
handle = commandLine.getOptionValue('i');
}
}

1840
dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImport.java
View File

File diff suppressed because it is too large Load Diff

130
dspace-api/src/main/java/org/dspace/content/MetadataExportServiceImpl.java Normal file
View File

@@ -0,0 +1,130 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.content;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import com.google.common.collect.Iterators;
import org.apache.logging.log4j.Logger;
import org.dspace.app.bulkedit.DSpaceCSV;
import org.dspace.content.service.ItemService;
import org.dspace.content.service.MetadataExportService;
import org.dspace.core.Constants;
import org.dspace.core.Context;
import org.dspace.handle.factory.HandleServiceFactory;
import org.springframework.beans.factory.annotation.Autowired;
public class MetadataExportServiceImpl implements MetadataExportService {
private static final Logger log = org.apache.logging.log4j.LogManager.getLogger(MetadataExportServiceImpl.class);
@Autowired
private ItemService itemService;
public DSpaceCSV handleExport(Context context, boolean exportAllItems, boolean exportAllMetadata, String handle)
throws Exception {
Iterator<Item> toExport = null;
if (!exportAllItems) {
log.info("Exporting whole repository WARNING: May take some time!");
toExport = itemService.findAll(context);
} else {
DSpaceObject dso = HandleServiceFactory.getInstance().getHandleService().resolveToObject(context, handle);
if (dso == null) {
throw new IllegalArgumentException(
"Item '" + handle + "' does not resolve to an item in your repository!");
}
if (dso.getType() == Constants.ITEM) {
log.info("Exporting item '" + dso.getName() + "' (" + handle + ")");
List<Item> item = new ArrayList<>();
item.add((Item) dso);
toExport = item.iterator();
} else if (dso.getType() == Constants.COLLECTION) {
log.info("Exporting collection '" + dso.getName() + "' (" + handle + ")");
Collection collection = (Collection) dso;
toExport = itemService.findByCollection(context, collection);
} else if (dso.getType() == Constants.COMMUNITY) {
log.info("Exporting community '" + dso.getName() + "' (" + handle + ")");
toExport = buildFromCommunity(context, (Community) dso);
} else {
throw new IllegalArgumentException("Error identifying '" + handle + "'");
}
}
DSpaceCSV csv = this.export(context, toExport, exportAllMetadata);
return csv;
}
/**
* Run the export
*
* @return the exported CSV lines
*/
public DSpaceCSV export(Context context, Iterator<Item> toExport, boolean exportAll) throws Exception {
Context.Mode originalMode = context.getCurrentMode();
context.setMode(Context.Mode.READ_ONLY);
// Process each item
DSpaceCSV csv = new DSpaceCSV(exportAll);
while (toExport.hasNext()) {
Item item = toExport.next();
csv.addItem(item);
context.uncacheEntity(item);
}
context.setMode(originalMode);
// Return the results
return csv;
}
public DSpaceCSV export(Context context, Community community, boolean exportAll) throws Exception {
return export(context, buildFromCommunity(context, community), exportAll);
}
/**
* Build an array list of item ids that are in a community (include sub-communities and collections)
*
* @param context DSpace context
* @param community The community to build from
* @return The list of item ids
* @throws SQLException if database error
*/
private Iterator<Item> buildFromCommunity(Context context, Community community)
throws SQLException {
// Add all the collections
List<Collection> collections = community.getCollections();
Iterator<Item> result = null;
for (Collection collection : collections) {
Iterator<Item> items = itemService.findByCollection(context, collection);
result = addItemsToResult(result, items);
}
// Add all the sub-communities
List<Community> communities = community.getSubcommunities();
for (Community subCommunity : communities) {
Iterator<Item> items = buildFromCommunity(context, subCommunity);
result = addItemsToResult(result, items);
}
return result;
}
private Iterator<Item> addItemsToResult(Iterator<Item> result, Iterator<Item> items) {
if (result == null) {
result = items;
} else {
result = Iterators.concat(result, items);
}
return result;
}
}

26
dspace-api/src/main/java/org/dspace/content/service/MetadataExportService.java Normal file
View File

@@ -0,0 +1,26 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.content.service;
import java.util.Iterator;
import org.dspace.app.bulkedit.DSpaceCSV;
import org.dspace.content.Community;
import org.dspace.content.Item;
import org.dspace.core.Context;
public interface MetadataExportService {
public DSpaceCSV handleExport(Context context, boolean exportAllItems, boolean exportAllMetadata, String handle)
throws Exception;
public DSpaceCSV export(Context context, Iterator<Item> toExport, boolean exportAll) throws Exception;
public DSpaceCSV export(Context context, Community community, boolean exportAll) throws Exception;
}

16
dspace-api/src/main/java/org/dspace/scripts/DSpaceRunnable.java
View File

@@ -7,10 +7,14 @@
*/
package org.dspace.scripts;
import java.io.InputStream;
import java.sql.SQLException;
import java.util.LinkedList;
import java.util.List;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.dspace.authorize.service.AuthorizeService;
@@ -73,6 +77,18 @@ public abstract class DSpaceRunnable implements Runnable {
return options;
}
public List<String> getFileNamesFromInputStreamOptions() {
List<String> fileNames = new LinkedList<>();
for (Option option : options.getOptions()) {
if (option.getType() == InputStream.class) {
fileNames.add(commandLine.getOptionValue(option.getOpt()));
}
}
return fileNames;
}
/**
* This method will take the primitive array of String objects that represent the parameters given to the String
* and it'll parse these into a CommandLine object that can be used by the script to retrieve the data

59
dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java
View File

@@ -7,6 +7,8 @@
*/
package org.dspace.scripts;
import java.io.IOException;
import java.io.InputStream;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collections;
@@ -17,8 +19,14 @@ import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.Logger;
import org.dspace.authorize.AuthorizeException;
import org.dspace.authorize.service.AuthorizeService;
import org.dspace.content.Bitstream;
import org.dspace.content.ProcessStatus;
import org.dspace.content.dao.ProcessDAO;
import org.dspace.content.service.BitstreamFormatService;
import org.dspace.content.service.BitstreamService;
import org.dspace.core.Constants;
import org.dspace.core.Context;
import org.dspace.core.LogManager;
import org.dspace.eperson.EPerson;
@@ -35,6 +43,15 @@ public class ProcessServiceImpl implements ProcessService {
@Autowired
private ProcessDAO processDAO;
@Autowired
private BitstreamService bitstreamService;
@Autowired
private BitstreamFormatService bitstreamFormatService;
@Autowired
private AuthorizeService authorizeService;
@Override
public Process create(Context context, EPerson ePerson, String scriptName,
List<DSpaceCommandLineParameter> parameters) throws SQLException {
@@ -112,6 +129,21 @@ public class ProcessServiceImpl implements ProcessService {
}
@Override
public void appendFile(Context context, Process process, InputStream is, String type, String fileName)
throws IOException, SQLException, AuthorizeException {
Bitstream bitstream = bitstreamService.create(context, is);
bitstream.setName(context, fileName);
bitstreamService.setFormat(context, bitstream, bitstreamFormatService.guessFormat(context, bitstream));
bitstreamService.addMetadata(context, bitstream, "process", "type", null, null, type);
authorizeService.addPolicy(context, bitstream, Constants.READ, context.getCurrentUser());
authorizeService.addPolicy(context, bitstream, Constants.WRITE, context.getCurrentUser());
authorizeService.addPolicy(context, bitstream, Constants.DELETE, context.getCurrentUser());
bitstreamService.update(context, bitstream);
process.addBitstream(bitstream);
update(context, process);
}
@Override
public void delete(Context context, Process process) throws SQLException {
processDAO.delete(context, process);
@@ -141,6 +173,33 @@ public class ProcessServiceImpl implements ProcessService {
return parameterList;
}
@Override
public Bitstream getBitstreamByName(Context context, Process process, String bitstreamName) {
for (Bitstream bitstream : getBitstreams(context, process, null)) {
if (StringUtils.equals(bitstream.getName(), bitstreamName)) {
return bitstream;
}
}
return null;
}
public List<Bitstream> getBitstreams(Context context, Process process, String type) {
List<Bitstream> allBitstreams = process.getBitstreams();
if (type == null) {
return allBitstreams;
} else {
List<Bitstream> filteredBitstreams = new ArrayList<>();
for (Bitstream bitstream : allBitstreams) {
if (StringUtils.equals(bitstreamService.getMetadata(bitstream, "process.type"), type)) {
filteredBitstreams.add(bitstream);
}
}
return filteredBitstreams;
}
}
public int countTotal(Context context) throws SQLException {
return processDAO.countRows(context);
}

9
dspace-api/src/main/java/org/dspace/scripts/handler/DSpaceRunnableHandler.java
View File

@@ -7,9 +7,13 @@
*/
package org.dspace.scripts.handler;
import java.io.IOException;
import java.io.InputStream;
import java.sql.SQLException;
import org.apache.commons.cli.Options;
import org.dspace.authorize.AuthorizeException;
import org.dspace.core.Context;
/**
* This is an interface meant to be implemented by any DSpaceRunnableHandler to specify specific execution methods
@@ -78,4 +82,9 @@ public interface DSpaceRunnableHandler {
* @param name The name of the script
*/
public void printHelp(Options options, String name);
public InputStream getFileStream(Context context, String fileName) throws IOException, AuthorizeException;
public void writeFilestream(Context context, String fileName, InputStream inputStream, String type)
throws IOException;
}

22
dspace-api/src/main/java/org/dspace/scripts/handler/impl/CommandLineDSpaceRunnableHandler.java
View File

@@ -7,9 +7,15 @@
*/
package org.dspace.scripts.handler.impl;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.Logger;
import org.dspace.core.Context;
import org.dspace.scripts.handler.DSpaceRunnableHandler;
/**
@@ -84,4 +90,20 @@ public class CommandLineDSpaceRunnableHandler implements DSpaceRunnableHandler {
formatter.printHelp(name, options);
}
}
@Override
public InputStream getFileStream(Context context, String fileName) throws IOException {
File file = new File(fileName);
if (!file.isFile()) {
return null;
}
return FileUtils.openInputStream(file);
}
@Override
public void writeFilestream(Context context, String fileName, InputStream inputStream, String type)
throws IOException {
File file = new File(fileName);
FileUtils.copyInputStreamToFile(inputStream, file);
}
}

11
dspace-api/src/main/java/org/dspace/scripts/service/ProcessService.java
View File

@@ -7,9 +7,13 @@
*/
package org.dspace.scripts.service;
import java.io.IOException;
import java.io.InputStream;
import java.sql.SQLException;
import java.util.List;
import org.dspace.authorize.AuthorizeException;
import org.dspace.content.Bitstream;
import org.dspace.core.Context;
import org.dspace.eperson.EPerson;
import org.dspace.scripts.DSpaceCommandLineParameter;
@@ -104,6 +108,9 @@ public interface ProcessService {
*/
public void complete(Context context, Process process) throws SQLException;
public void appendFile(Context context, Process process, InputStream is, String type, String fileName)
throws IOException, SQLException, AuthorizeException;
/**
* This method will delete the given Process object from the database
* @param context The relevant DSpace context
@@ -128,6 +135,10 @@ public interface ProcessService {
*/
public List<DSpaceCommandLineParameter> getParameters(Process process);
public Bitstream getBitstreamByName(Context context, Process process, String bitstreamName);
public List<Bitstream> getBitstreams(Context context, Process process, String type);
/**
* Returns the total amount of Process objects in the dataase
* @param context The relevant DSpace context

9
dspace-server-webapp/src/main/java/org/dspace/app/rest/ScriptProcessesController.java
View File

@@ -7,6 +7,8 @@
*/
package org.dspace.app.rest;
import java.util.List;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.dspace.app.rest.link.HalLinkService;
@@ -24,7 +26,9 @@ import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
/**
* This controller adds additional subresource methods to allow connecting scripts with processes
@@ -53,12 +57,13 @@ public class ScriptProcessesController {
*/
@RequestMapping(method = RequestMethod.POST)
@PreAuthorize("hasAuthority('ADMIN')")
public ResponseEntity<ResourceSupport> startProcess(@PathVariable(name = "name") String scriptName)
public ResponseEntity<ResourceSupport> startProcess(@PathVariable(name = "name") String scriptName,
@RequestParam(name = "file") List<MultipartFile> files)
throws Exception {
if (log.isTraceEnabled()) {
log.trace("Starting Process for Script with name: " + scriptName);
}
ProcessRest processRest = scriptRestRepository.startProcess(scriptName);
ProcessRest processRest = scriptRestRepository.startProcess(scriptName, files);
ProcessResource processResource = new ProcessResource(processRest, utils, null);
halLinkService.addLinks(processResource);
return ControllerUtils.toResponseEntity(HttpStatus.ACCEPTED, null, processResource);

5
dspace-server-webapp/src/main/java/org/dspace/app/rest/exception/DSpaceApiExceptionControllerAdvice.java
View File

@@ -29,6 +29,7 @@ import org.springframework.web.bind.annotation.ControllerAdvice;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.context.request.WebRequest;
import org.springframework.web.multipart.MultipartException;
import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler;
/**
@@ -54,8 +55,8 @@ public class DSpaceApiExceptionControllerAdvice extends ResponseEntityExceptionH
}
}
@ExceptionHandler(IllegalArgumentException.class)
protected void handleIllegalArgumentException(HttpServletRequest request, HttpServletResponse response,
@ExceptionHandler({IllegalArgumentException.class, MultipartException.class})
protected void handleWrongRequestException(HttpServletRequest request, HttpServletResponse response,
Exception ex) throws IOException {
sendErrorResponse(request, response, ex, ex.getMessage(), HttpServletResponse.SC_BAD_REQUEST);
}

41
dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ScriptRestRepository.java
View File

@@ -24,6 +24,7 @@ import org.dspace.app.rest.converter.DSpaceRunnableParameterConverter;
import org.dspace.app.rest.converter.ScriptConverter;
import org.dspace.app.rest.converter.processes.ProcessConverter;
import org.dspace.app.rest.exception.DSpaceBadRequestException;
import org.dspace.app.rest.exception.UnprocessableEntityException;
import org.dspace.app.rest.model.ParameterValueRest;
import org.dspace.app.rest.model.ProcessRest;
import org.dspace.app.rest.model.ScriptRest;
@@ -40,6 +41,7 @@ import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.security.access.AccessDeniedException;
import org.springframework.stereotype.Component;
import org.springframework.web.multipart.MultipartFile;
/**
* This is the REST repository dealing with the Script logic
@@ -103,7 +105,8 @@ public class ScriptRestRepository extends DSpaceRestRepository<ScriptRest, Strin
* @throws SQLException If something goes wrong
* @throws IOException If something goes wrong
*/
public ProcessRest startProcess(String scriptName) throws SQLException, IOException, AuthorizeException {
public ProcessRest startProcess(String scriptName,
List<MultipartFile> files) throws SQLException, IOException, AuthorizeException {
Context context = obtainContext();
String properties = requestService.getCurrentRequest().getServletRequest().getParameter("properties");
List<DSpaceCommandLineParameter> dSpaceCommandLineParameters =
@@ -119,7 +122,7 @@ public class ScriptRestRepository extends DSpaceRestRepository<ScriptRest, Strin
context.getCurrentUser(), scriptName, dSpaceCommandLineParameters);
List<String> args = constructArgs(dSpaceCommandLineParameters);
try {
runDSpaceScript(scriptToExecute, restDSpaceRunnableHandler, args);
runDSpaceScript(files, context, scriptToExecute, restDSpaceRunnableHandler, args);
context.complete();
return processConverter.fromModel(restDSpaceRunnableHandler.getProcess());
} catch (SQLException e) {
@@ -167,10 +170,13 @@ public class ScriptRestRepository extends DSpaceRestRepository<ScriptRest, Strin
return args;
}
private void runDSpaceScript(DSpaceRunnable scriptToExecute,
RestDSpaceRunnableHandler restDSpaceRunnableHandler, List<String> args) {
private void runDSpaceScript(List<MultipartFile> files, Context context, DSpaceRunnable scriptToExecute,
RestDSpaceRunnableHandler restDSpaceRunnableHandler, List<String> args)
throws IOException {
try {
scriptToExecute.initialize(args.toArray(new String[0]), restDSpaceRunnableHandler);
checkFileNames(scriptToExecute, files);
processFiles(context, restDSpaceRunnableHandler, files);
restDSpaceRunnableHandler.schedule(scriptToExecute);
} catch (ParseException e) {
scriptToExecute.printHelp();
@@ -181,4 +187,31 @@ public class ScriptRestRepository extends DSpaceRestRepository<ScriptRest, Strin
}
}
private void processFiles(Context context, RestDSpaceRunnableHandler restDSpaceRunnableHandler,
List<MultipartFile> files)
throws IOException {
for (MultipartFile file : files) {
restDSpaceRunnableHandler
.writeFilestream(context, file.getOriginalFilename(), file.getInputStream(), "inputfile");
}
}
private void checkFileNames(DSpaceRunnable scriptToExecute, List<MultipartFile> files) {
List<String> fileNames = new LinkedList<>();
for (MultipartFile file : files) {
String fileName = file.getOriginalFilename();
if (fileNames.contains(fileName)) {
throw new UnprocessableEntityException("There are two files with the same name: " + fileName);
} else {
fileNames.add(fileName);
}
}
List<String> fileNamesFromOptions = scriptToExecute.getFileNamesFromInputStreamOptions();
if (!fileNames.containsAll(fileNamesFromOptions)) {
throw new UnprocessableEntityException("Files given in properties aren't all present in the request");
}
}
}

37
dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java
View File

@@ -7,6 +7,8 @@
*/
package org.dspace.app.rest.scripts.handler.impl;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.sql.SQLException;
@@ -16,8 +18,12 @@ import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.logging.log4j.Logger;
import org.dspace.authorize.AuthorizeException;
import org.dspace.content.Bitstream;
import org.dspace.content.ProcessStatus;
import org.dspace.content.factory.ContentServiceFactory;
import org.dspace.content.factory.ProcessServiceFactory;
import org.dspace.content.service.BitstreamService;
import org.dspace.core.Context;
import org.dspace.eperson.EPerson;
import org.dspace.scripts.DSpaceCommandLineParameter;
@@ -25,6 +31,8 @@ import org.dspace.scripts.DSpaceRunnable;
import org.dspace.scripts.Process;
import org.dspace.scripts.handler.DSpaceRunnableHandler;
import org.dspace.scripts.service.ProcessService;
import org.dspace.utils.DSpace;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
/**
* The {@link DSpaceRunnableHandler} dealing with Scripts started from the REST api
@@ -34,6 +42,7 @@ public class RestDSpaceRunnableHandler implements DSpaceRunnableHandler {
.getLogger(RestDSpaceRunnableHandler.class);
private ProcessService processService = ProcessServiceFactory.getInstance().getProcessService();
private BitstreamService bitstreamService = ContentServiceFactory.getInstance().getBitstreamService();
private Integer processId;
private String scriptName;
@@ -176,6 +185,29 @@ public class RestDSpaceRunnableHandler implements DSpaceRunnableHandler {
}
}
@Override
public InputStream getFileStream(Context context, String fileName) throws IOException, AuthorizeException {
try {
Process process = processService.find(context, processId);
Bitstream bitstream = processService.getBitstreamByName(context, process, fileName);
return bitstreamService.retrieve(context, bitstream);
} catch (SQLException sqlException) {
log.error("SQL exception while attempting to find process", sqlException);
}
return null;
}
@Override
public void writeFilestream(Context context, String fileName, InputStream inputStream, String type)
throws IOException {
try {
Process process = processService.find(context, processId);
processService.appendFile(context, process, inputStream, type, fileName);
} catch (SQLException | AuthorizeException exception) {
log.error("Exception occurred while attempting to find process", exception);
}
}
/**
* This method will return the process created by this handler
* @return The Process database object created by this handler
@@ -200,6 +232,9 @@ public class RestDSpaceRunnableHandler implements DSpaceRunnableHandler {
* @param script The script to be ran
*/
public void schedule(DSpaceRunnable script) {
ThreadPoolTaskExecutor taskExecutor = new DSpace().getServiceManager()
.getServiceByName("dspaceRunnableThreadExecutor",
ThreadPoolTaskExecutor.class);
Context context = new Context();
try {
Process process = processService.find(context, processId);
@@ -213,6 +248,6 @@ public class RestDSpaceRunnableHandler implements DSpaceRunnableHandler {
context.abort();
}
}
script.run();
taskExecutor.execute(script);
}
}

9
dspace/config/spring/rest/scripts.xml Normal file
View File

@@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">
<bean id="dspaceRunnableThreadExecutor" class="org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor">
<property name="corePoolSize" value="5"/>
</bean>
</beans>