hazza/DSpace
1
0
Fork 0
mirror of https://github.com/DSpace/DSpace.git synced 2025-10-17 15:03:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implementing file import and export scripts

Browse Source
This commit is contained in:
Raf Ponsaerts
2019-11-21 13:22:09 +01:00
parent 0ea677914f
commit 08c7da4446
15 changed files with 1501 additions and 1036 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
dspace-api/src/main/java/org/dspace/app/bulkedit/DSpaceCSV.java
View File

@@ -13,6 +13,7 @@ import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.io.OutputStreamWriter; import java.io.OutputStreamWriter;
import java.io.Serializable; import java.io.Serializable;
@@ -27,7 +28,9 @@ import java.util.UUID;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.pdfbox.util.Charsets;
import org.dspace.authority.AuthorityValue; import org.dspace.authority.AuthorityValue;
import org.dspace.authority.factory.AuthorityServiceFactory; import org.dspace.authority.factory.AuthorityServiceFactory;
import org.dspace.authority.service.AuthorityValueService; import org.dspace.authority.service.AuthorityValueService;
@@ -637,6 +640,14 @@ public class DSpaceCSV implements Serializable {
out.close(); out.close();
} }
public InputStream getInputStream() {
StringBuilder stringBuilder = new StringBuilder();
for (String csvLine : getCSVLinesAsStringArray()) {
stringBuilder.append(csvLine + "\n");
}
return IOUtils.toInputStream(stringBuilder.toString(), Charsets.UTF_8);
}
/** /**
* Is it Ok to export this value? When exportAll is set to false, we don't export * Is it Ok to export this value? When exportAll is set to false, we don't export
* some of the metadata elements. * some of the metadata elements.

288
dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExport.java
View File

@@ -7,272 +7,84 @@
*/ */
package org.dspace.app.bulkedit; package org.dspace.app.bulkedit;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import com.google.common.collect.Iterators;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options; import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException; import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser; import org.dspace.app.bulkedit.DSpaceCSV;
import org.dspace.content.Collection; import org.dspace.content.service.MetadataExportService;
import org.dspace.content.Community;
import org.dspace.content.DSpaceObject;
import org.dspace.content.Item;
import org.dspace.content.factory.ContentServiceFactory;
import org.dspace.content.service.ItemService;
import org.dspace.core.Constants;
import org.dspace.core.Context; import org.dspace.core.Context;
import org.dspace.handle.factory.HandleServiceFactory; import org.dspace.scripts.DSpaceRunnable;
import org.springframework.beans.factory.annotation.Autowired;
/** /**
* Metadata exporter to allow the batch export of metadata into a file * Metadata exporter to allow the batch export of metadata into a file
* *
* @author Stuart Lewis * @author Stuart Lewis
*/ */
public class MetadataExport { public class MetadataExport extends DSpaceRunnable {
/**
* The items to export
*/
protected Iterator<Item> toExport;
protected ItemService itemService; private Context c = null;
private boolean help = false;
private String filename = null;
private String handle = null;
private boolean exportAllMetadata = false;
private boolean exportAllItems = false;
@Autowired
private MetadataExportService metadataExportService;
protected Context context; protected Context context;
/** private MetadataExport() {
* Whether to export all metadata, or just normally edited metadata Options options = constructOptions();
*/ this.options = options;
protected boolean exportAll;
protected MetadataExport() {
itemService = ContentServiceFactory.getInstance().getItemService();
} }
/** private Options constructOptions() {
* Set up a new metadata export
*
* @param c The Context
* @param toExport The ItemIterator of items to export
* @param exportAll whether to export all metadata or not (include handle, provenance etc)
*/
public MetadataExport(Context c, Iterator<Item> toExport, boolean exportAll) {
itemService = ContentServiceFactory.getInstance().getItemService();
// Store the export settings
this.toExport = toExport;
this.exportAll = exportAll;
this.context = c;
}
/**
* Method to export a community (and sub-communities and collections)
*
* @param c The Context
* @param toExport The Community to export
* @param exportAll whether to export all metadata or not (include handle, provenance etc)
*/
public MetadataExport(Context c, Community toExport, boolean exportAll) {
itemService = ContentServiceFactory.getInstance().getItemService();
try {
// Try to export the community
this.toExport = buildFromCommunity(c, toExport, 0);
this.exportAll = exportAll;
this.context = c;
} catch (SQLException sqle) {
// Something went wrong...
System.err.println("Error running exporter:");
sqle.printStackTrace(System.err);
System.exit(1);
}
}
/**
* Build an array list of item ids that are in a community (include sub-communities and collections)
*
* @param context DSpace context
* @param community The community to build from
* @param indent How many spaces to use when writing out the names of items added
* @return The list of item ids
* @throws SQLException if database error
*/
protected Iterator<Item> buildFromCommunity(Context context, Community community, int indent)
throws SQLException {
// Add all the collections
List<Collection> collections = community.getCollections();
Iterator<Item> result = null;
for (Collection collection : collections) {
for (int i = 0; i < indent; i++) {
System.out.print(" ");
}
Iterator<Item> items = itemService.findByCollection(context, collection);
result = addItemsToResult(result, items);
}
// Add all the sub-communities
List<Community> communities = community.getSubcommunities();
for (Community subCommunity : communities) {
for (int i = 0; i < indent; i++) {
System.out.print(" ");
}
Iterator<Item> items = buildFromCommunity(context, subCommunity, indent + 1);
result = addItemsToResult(result, items);
}
return result;
}
private Iterator<Item> addItemsToResult(Iterator<Item> result, Iterator<Item> items) {
if (result == null) {
result = items;
} else {
result = Iterators.concat(result, items);
}
return result;
}
/**
* Run the export
*
* @return the exported CSV lines
*/
public DSpaceCSV export() {
try {
Context.Mode originalMode = context.getCurrentMode();
context.setMode(Context.Mode.READ_ONLY);
// Process each item
DSpaceCSV csv = new DSpaceCSV(exportAll);
while (toExport.hasNext()) {
Item item = toExport.next();
csv.addItem(item);
context.uncacheEntity(item);
}
context.setMode(originalMode);
// Return the results
return csv;
} catch (Exception e) {
// Something went wrong...
System.err.println("Error exporting to CSV:");
e.printStackTrace();
return null;
}
}
/**
* Print the help message
*
* @param options The command line options the user gave
* @param exitCode the system exit code to use
*/
private static void printHelp(Options options, int exitCode) {
// print the help message
HelpFormatter myhelp = new HelpFormatter();
myhelp.printHelp("MetadataExport\n", options);
System.out.println("\nfull export: metadataexport -f filename");
System.out.println("partial export: metadataexport -i handle -f filename");
System.exit(exitCode);
}
/**
* main method to run the metadata exporter
*
* @param argv the command line arguments given
* @throws Exception if error occurs
*/
public static void main(String[] argv) throws Exception {
// Create an options object and populate it
CommandLineParser parser = new PosixParser();
Options options = new Options(); Options options = new Options();
options.addOption("i", "id", true, "ID or handle of thing to export (item, collection, or community)"); options.addOption("i", "id", true, "ID or handle of thing to export (item, collection, or community)");
options.addOption("f", "file", true, "destination where you want file written"); options.addOption("f", "file", true, "destination where you want file written");
options.getOption("f").setRequired(true);
options.addOption("a", "all", false, options.addOption("a", "all", false,
"include all metadata fields that are not normally changed (e.g. provenance)"); "include all metadata fields that are not normally changed (e.g. provenance)");
options.addOption("h", "help", false, "help"); options.addOption("h", "help", false, "help");
CommandLine line = null; return options;
try {
line = parser.parse(options, argv);
} catch (ParseException pe) {
System.err.println("Error with commands.");
printHelp(options, 1);
System.exit(0);
} }
if (line.hasOption('h')) { public void internalRun() throws Exception {
printHelp(options, 0); if (help) {
handler.logInfo("\nfull export: metadataexport -f filename");
handler.logInfo("partial export: metadataexport -i handle -f filename");
printHelp();
return;
} }
// Check a filename is given DSpaceCSV dSpaceCSV = metadataExportService.handleExport(c, exportAllItems, exportAllMetadata, handle);
if (!line.hasOption('f')) { handler.writeFilestream(c, filename, dSpaceCSV.getInputStream(), "exportCSV");
System.err.println("Required parameter -f missing!");
printHelp(options, 1);
}
String filename = line.getOptionValue('f');
// Create a context
Context c = new Context(Context.Mode.READ_ONLY);
c.turnOffAuthorisationSystem();
// The things we'll export
Iterator<Item> toExport = null;
MetadataExport exporter = null;
// Export everything?
boolean exportAll = line.hasOption('a');
ContentServiceFactory contentServiceFactory = ContentServiceFactory.getInstance();
// Check we have an item OK
ItemService itemService = contentServiceFactory.getItemService();
if (!line.hasOption('i')) {
System.out.println("Exporting whole repository WARNING: May take some time!");
exporter = new MetadataExport(c, itemService.findAll(c), exportAll);
} else {
String handle = line.getOptionValue('i');
DSpaceObject dso = HandleServiceFactory.getInstance().getHandleService().resolveToObject(c, handle);
if (dso == null) {
System.err.println("Item '" + handle + "' does not resolve to an item in your repository!");
printHelp(options, 1);
}
if (dso.getType() == Constants.ITEM) {
System.out.println("Exporting item '" + dso.getName() + "' (" + handle + ")");
List<Item> item = new ArrayList<>();
item.add((Item) dso);
exporter = new MetadataExport(c, item.iterator(), exportAll);
} else if (dso.getType() == Constants.COLLECTION) {
System.out.println("Exporting collection '" + dso.getName() + "' (" + handle + ")");
Collection collection = (Collection) dso;
toExport = itemService.findByCollection(c, collection);
exporter = new MetadataExport(c, toExport, exportAll);
} else if (dso.getType() == Constants.COMMUNITY) {
System.out.println("Exporting community '" + dso.getName() + "' (" + handle + ")");
exporter = new MetadataExport(c, (Community) dso, exportAll);
} else {
System.err.println("Error identifying '" + handle + "'");
System.exit(1);
}
}
// Perform the export
DSpaceCSV csv = exporter.export();
// Save the files to the file
csv.save(filename);
// Finish off and tidy up
c.restoreAuthSystemState(); c.restoreAuthSystemState();
c.complete(); c.complete();
} }
public void setup() throws ParseException {
c = new Context();
c.turnOffAuthorisationSystem();
if (commandLine.hasOption('h')) {
help = true;
}
// Check a filename is given
if (!commandLine.hasOption('f')) {
throw new ParseException("Required parameter -f missing!");
}
filename = commandLine.getOptionValue('f');
exportAllMetadata = commandLine.hasOption('a');
if (commandLine.hasOption('i')) {
exportAllItems = true;
}
handle = commandLine.getOptionValue('i');
}
} }

1840
dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImport.java
View File

File diff suppressed because it is too large Load Diff

130
dspace-api/src/main/java/org/dspace/content/MetadataExportServiceImpl.java Normal file
View File

@@ -0,0 +1,130 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.content;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import com.google.common.collect.Iterators;
import org.apache.logging.log4j.Logger;
import org.dspace.app.bulkedit.DSpaceCSV;
import org.dspace.content.service.ItemService;
import org.dspace.content.service.MetadataExportService;
import org.dspace.core.Constants;
import org.dspace.core.Context;
import org.dspace.handle.factory.HandleServiceFactory;
import org.springframework.beans.factory.annotation.Autowired;
public class MetadataExportServiceImpl implements MetadataExportService {
private static final Logger log = org.apache.logging.log4j.LogManager.getLogger(MetadataExportServiceImpl.class);
@Autowired
private ItemService itemService;
public DSpaceCSV handleExport(Context context, boolean exportAllItems, boolean exportAllMetadata, String handle)
throws Exception {
Iterator<Item> toExport = null;
if (!exportAllItems) {
log.info("Exporting whole repository WARNING: May take some time!");
toExport = itemService.findAll(context);
} else {
DSpaceObject dso = HandleServiceFactory.getInstance().getHandleService().resolveToObject(context, handle);
if (dso == null) {
throw new IllegalArgumentException(
"Item '" + handle + "' does not resolve to an item in your repository!");
}
if (dso.getType() == Constants.ITEM) {
log.info("Exporting item '" + dso.getName() + "' (" + handle + ")");
List<Item> item = new ArrayList<>();
item.add((Item) dso);
toExport = item.iterator();
} else if (dso.getType() == Constants.COLLECTION) {
log.info("Exporting collection '" + dso.getName() + "' (" + handle + ")");
Collection collection = (Collection) dso;
toExport = itemService.findByCollection(context, collection);
} else if (dso.getType() == Constants.COMMUNITY) {
log.info("Exporting community '" + dso.getName() + "' (" + handle + ")");
toExport = buildFromCommunity(context, (Community) dso);
} else {
throw new IllegalArgumentException("Error identifying '" + handle + "'");
}
}
DSpaceCSV csv = this.export(context, toExport, exportAllMetadata);
return csv;
}
/**
* Run the export
*
* @return the exported CSV lines
*/
public DSpaceCSV export(Context context, Iterator<Item> toExport, boolean exportAll) throws Exception {
Context.Mode originalMode = context.getCurrentMode();
context.setMode(Context.Mode.READ_ONLY);
// Process each item
DSpaceCSV csv = new DSpaceCSV(exportAll);
while (toExport.hasNext()) {
Item item = toExport.next();
csv.addItem(item);
context.uncacheEntity(item);
}
context.setMode(originalMode);
// Return the results
return csv;
}
public DSpaceCSV export(Context context, Community community, boolean exportAll) throws Exception {
return export(context, buildFromCommunity(context, community), exportAll);
}
/**
* Build an array list of item ids that are in a community (include sub-communities and collections)
*
* @param context DSpace context
* @param community The community to build from
* @return The list of item ids
* @throws SQLException if database error
*/
private Iterator<Item> buildFromCommunity(Context context, Community community)
throws SQLException {
// Add all the collections
List<Collection> collections = community.getCollections();
Iterator<Item> result = null;
for (Collection collection : collections) {
Iterator<Item> items = itemService.findByCollection(context, collection);
result = addItemsToResult(result, items);
}
// Add all the sub-communities
List<Community> communities = community.getSubcommunities();
for (Community subCommunity : communities) {
Iterator<Item> items = buildFromCommunity(context, subCommunity);
result = addItemsToResult(result, items);
}
return result;
}
private Iterator<Item> addItemsToResult(Iterator<Item> result, Iterator<Item> items) {
if (result == null) {
result = items;
} else {
result = Iterators.concat(result, items);
}
return result;
}
}

26
dspace-api/src/main/java/org/dspace/content/service/MetadataExportService.java Normal file
View File

@@ -0,0 +1,26 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.content.service;
import java.util.Iterator;
import org.dspace.app.bulkedit.DSpaceCSV;
import org.dspace.content.Community;
import org.dspace.content.Item;
import org.dspace.core.Context;
public interface MetadataExportService {
public DSpaceCSV handleExport(Context context, boolean exportAllItems, boolean exportAllMetadata, String handle)
throws Exception;
public DSpaceCSV export(Context context, Iterator<Item> toExport, boolean exportAll) throws Exception;
public DSpaceCSV export(Context context, Community community, boolean exportAll) throws Exception;
}

16
dspace-api/src/main/java/org/dspace/scripts/DSpaceRunnable.java
View File

@@ -7,10 +7,14 @@
*/ */
package org.dspace.scripts; package org.dspace.scripts;
import java.io.InputStream;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.LinkedList;
import java.util.List;
import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options; import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException; import org.apache.commons.cli.ParseException;
import org.dspace.authorize.service.AuthorizeService; import org.dspace.authorize.service.AuthorizeService;
@@ -73,6 +77,18 @@ public abstract class DSpaceRunnable implements Runnable {
return options; return options;
} }
public List<String> getFileNamesFromInputStreamOptions() {
List<String> fileNames = new LinkedList<>();
for (Option option : options.getOptions()) {
if (option.getType() == InputStream.class) {
fileNames.add(commandLine.getOptionValue(option.getOpt()));
}
}
return fileNames;
}
/** /**
* This method will take the primitive array of String objects that represent the parameters given to the String * This method will take the primitive array of String objects that represent the parameters given to the String
* and it'll parse these into a CommandLine object that can be used by the script to retrieve the data * and it'll parse these into a CommandLine object that can be used by the script to retrieve the data

59
dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java
View File

@@ -7,6 +7,8 @@
*/ */
package org.dspace.scripts; package org.dspace.scripts;
import java.io.IOException;
import java.io.InputStream;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
@@ -17,8 +19,14 @@ import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.dspace.authorize.AuthorizeException;
import org.dspace.authorize.service.AuthorizeService;
import org.dspace.content.Bitstream;
import org.dspace.content.ProcessStatus; import org.dspace.content.ProcessStatus;
import org.dspace.content.dao.ProcessDAO; import org.dspace.content.dao.ProcessDAO;
import org.dspace.content.service.BitstreamFormatService;
import org.dspace.content.service.BitstreamService;
import org.dspace.core.Constants;
import org.dspace.core.Context; import org.dspace.core.Context;
import org.dspace.core.LogManager; import org.dspace.core.LogManager;
import org.dspace.eperson.EPerson; import org.dspace.eperson.EPerson;
@@ -35,6 +43,15 @@ public class ProcessServiceImpl implements ProcessService {
@Autowired @Autowired
private ProcessDAO processDAO; private ProcessDAO processDAO;
@Autowired
private BitstreamService bitstreamService;
@Autowired
private BitstreamFormatService bitstreamFormatService;
@Autowired
private AuthorizeService authorizeService;
@Override @Override
public Process create(Context context, EPerson ePerson, String scriptName, public Process create(Context context, EPerson ePerson, String scriptName,
List<DSpaceCommandLineParameter> parameters) throws SQLException { List<DSpaceCommandLineParameter> parameters) throws SQLException {
@@ -112,6 +129,21 @@ public class ProcessServiceImpl implements ProcessService {
} }
@Override
public void appendFile(Context context, Process process, InputStream is, String type, String fileName)
throws IOException, SQLException, AuthorizeException {
Bitstream bitstream = bitstreamService.create(context, is);
bitstream.setName(context, fileName);
bitstreamService.setFormat(context, bitstream, bitstreamFormatService.guessFormat(context, bitstream));
bitstreamService.addMetadata(context, bitstream, "process", "type", null, null, type);
authorizeService.addPolicy(context, bitstream, Constants.READ, context.getCurrentUser());
authorizeService.addPolicy(context, bitstream, Constants.WRITE, context.getCurrentUser());
authorizeService.addPolicy(context, bitstream, Constants.DELETE, context.getCurrentUser());
bitstreamService.update(context, bitstream);
process.addBitstream(bitstream);
update(context, process);
}
@Override @Override
public void delete(Context context, Process process) throws SQLException { public void delete(Context context, Process process) throws SQLException {
processDAO.delete(context, process); processDAO.delete(context, process);
@@ -141,6 +173,33 @@ public class ProcessServiceImpl implements ProcessService {
return parameterList; return parameterList;
} }
@Override
public Bitstream getBitstreamByName(Context context, Process process, String bitstreamName) {
for (Bitstream bitstream : getBitstreams(context, process, null)) {
if (StringUtils.equals(bitstream.getName(), bitstreamName)) {
return bitstream;
}
}
return null;
}
public List<Bitstream> getBitstreams(Context context, Process process, String type) {
List<Bitstream> allBitstreams = process.getBitstreams();
if (type == null) {
return allBitstreams;
} else {
List<Bitstream> filteredBitstreams = new ArrayList<>();
for (Bitstream bitstream : allBitstreams) {
if (StringUtils.equals(bitstreamService.getMetadata(bitstream, "process.type"), type)) {
filteredBitstreams.add(bitstream);
}
}
return filteredBitstreams;
}
}
public int countTotal(Context context) throws SQLException { public int countTotal(Context context) throws SQLException {
return processDAO.countRows(context); return processDAO.countRows(context);
} }

9
dspace-api/src/main/java/org/dspace/scripts/handler/DSpaceRunnableHandler.java
View File

@@ -7,9 +7,13 @@
*/ */
package org.dspace.scripts.handler; package org.dspace.scripts.handler;
import java.io.IOException;
import java.io.InputStream;
import java.sql.SQLException; import java.sql.SQLException;
import org.apache.commons.cli.Options; import org.apache.commons.cli.Options;
import org.dspace.authorize.AuthorizeException;
import org.dspace.core.Context;
/** /**
* This is an interface meant to be implemented by any DSpaceRunnableHandler to specify specific execution methods * This is an interface meant to be implemented by any DSpaceRunnableHandler to specify specific execution methods
@@ -78,4 +82,9 @@ public interface DSpaceRunnableHandler {
* @param name The name of the script * @param name The name of the script
*/ */
public void printHelp(Options options, String name); public void printHelp(Options options, String name);
public InputStream getFileStream(Context context, String fileName) throws IOException, AuthorizeException;
public void writeFilestream(Context context, String fileName, InputStream inputStream, String type)
throws IOException;
} }

22
dspace-api/src/main/java/org/dspace/scripts/handler/impl/CommandLineDSpaceRunnableHandler.java
View File

@@ -7,9 +7,15 @@
*/ */
package org.dspace.scripts.handler.impl; package org.dspace.scripts.handler.impl;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options; import org.apache.commons.cli.Options;
import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.dspace.core.Context;
import org.dspace.scripts.handler.DSpaceRunnableHandler; import org.dspace.scripts.handler.DSpaceRunnableHandler;
/** /**
@@ -84,4 +90,20 @@ public class CommandLineDSpaceRunnableHandler implements DSpaceRunnableHandler {
formatter.printHelp(name, options); formatter.printHelp(name, options);
} }
} }
@Override
public InputStream getFileStream(Context context, String fileName) throws IOException {
File file = new File(fileName);
if (!file.isFile()) {
return null;
}
return FileUtils.openInputStream(file);
}
@Override
public void writeFilestream(Context context, String fileName, InputStream inputStream, String type)
throws IOException {
File file = new File(fileName);
FileUtils.copyInputStreamToFile(inputStream, file);
}
} }

11
dspace-api/src/main/java/org/dspace/scripts/service/ProcessService.java
View File

@@ -7,9 +7,13 @@
*/ */
package org.dspace.scripts.service; package org.dspace.scripts.service;
import java.io.IOException;
import java.io.InputStream;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.List; import java.util.List;
import org.dspace.authorize.AuthorizeException;
import org.dspace.content.Bitstream;
import org.dspace.core.Context; import org.dspace.core.Context;
import org.dspace.eperson.EPerson; import org.dspace.eperson.EPerson;
import org.dspace.scripts.DSpaceCommandLineParameter; import org.dspace.scripts.DSpaceCommandLineParameter;
@@ -104,6 +108,9 @@ public interface ProcessService {
*/ */
public void complete(Context context, Process process) throws SQLException; public void complete(Context context, Process process) throws SQLException;
public void appendFile(Context context, Process process, InputStream is, String type, String fileName)
throws IOException, SQLException, AuthorizeException;
/** /**
* This method will delete the given Process object from the database * This method will delete the given Process object from the database
* @param context The relevant DSpace context * @param context The relevant DSpace context
@@ -128,6 +135,10 @@ public interface ProcessService {
*/ */
public List<DSpaceCommandLineParameter> getParameters(Process process); public List<DSpaceCommandLineParameter> getParameters(Process process);
public Bitstream getBitstreamByName(Context context, Process process, String bitstreamName);
public List<Bitstream> getBitstreams(Context context, Process process, String type);
/** /**
* Returns the total amount of Process objects in the dataase * Returns the total amount of Process objects in the dataase
* @param context The relevant DSpace context * @param context The relevant DSpace context

9
dspace-server-webapp/src/main/java/org/dspace/app/rest/ScriptProcessesController.java
View File

@@ -7,6 +7,8 @@
*/ */
package org.dspace.app.rest; package org.dspace.app.rest;
import java.util.List;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.dspace.app.rest.link.HalLinkService; import org.dspace.app.rest.link.HalLinkService;
@@ -24,7 +26,9 @@ import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
/** /**
* This controller adds additional subresource methods to allow connecting scripts with processes * This controller adds additional subresource methods to allow connecting scripts with processes
@@ -53,12 +57,13 @@ public class ScriptProcessesController {
*/ */
@RequestMapping(method = RequestMethod.POST) @RequestMapping(method = RequestMethod.POST)
@PreAuthorize("hasAuthority('ADMIN')") @PreAuthorize("hasAuthority('ADMIN')")
public ResponseEntity<ResourceSupport> startProcess(@PathVariable(name = "name") String scriptName) public ResponseEntity<ResourceSupport> startProcess(@PathVariable(name = "name") String scriptName,
@RequestParam(name = "file") List<MultipartFile> files)
throws Exception { throws Exception {
if (log.isTraceEnabled()) { if (log.isTraceEnabled()) {
log.trace("Starting Process for Script with name: " + scriptName); log.trace("Starting Process for Script with name: " + scriptName);
} }
ProcessRest processRest = scriptRestRepository.startProcess(scriptName); ProcessRest processRest = scriptRestRepository.startProcess(scriptName, files);
ProcessResource processResource = new ProcessResource(processRest, utils, null); ProcessResource processResource = new ProcessResource(processRest, utils, null);
halLinkService.addLinks(processResource); halLinkService.addLinks(processResource);
return ControllerUtils.toResponseEntity(HttpStatus.ACCEPTED, null, processResource); return ControllerUtils.toResponseEntity(HttpStatus.ACCEPTED, null, processResource);

5
dspace-server-webapp/src/main/java/org/dspace/app/rest/exception/DSpaceApiExceptionControllerAdvice.java
View File

@@ -29,6 +29,7 @@ import org.springframework.web.bind.annotation.ControllerAdvice;
import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.context.request.WebRequest; import org.springframework.web.context.request.WebRequest;
import org.springframework.web.multipart.MultipartException;
import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler; import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler;
/** /**
@@ -54,8 +55,8 @@ public class DSpaceApiExceptionControllerAdvice extends ResponseEntityExceptionH
} }
} }
@ExceptionHandler(IllegalArgumentException.class) @ExceptionHandler({IllegalArgumentException.class, MultipartException.class})
protected void handleIllegalArgumentException(HttpServletRequest request, HttpServletResponse response, protected void handleWrongRequestException(HttpServletRequest request, HttpServletResponse response,
Exception ex) throws IOException { Exception ex) throws IOException {
sendErrorResponse(request, response, ex, ex.getMessage(), HttpServletResponse.SC_BAD_REQUEST); sendErrorResponse(request, response, ex, ex.getMessage(), HttpServletResponse.SC_BAD_REQUEST);
} }

41
dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ScriptRestRepository.java
View File

@@ -24,6 +24,7 @@ import org.dspace.app.rest.converter.DSpaceRunnableParameterConverter;
import org.dspace.app.rest.converter.ScriptConverter; import org.dspace.app.rest.converter.ScriptConverter;
import org.dspace.app.rest.converter.processes.ProcessConverter; import org.dspace.app.rest.converter.processes.ProcessConverter;
import org.dspace.app.rest.exception.DSpaceBadRequestException; import org.dspace.app.rest.exception.DSpaceBadRequestException;
import org.dspace.app.rest.exception.UnprocessableEntityException;
import org.dspace.app.rest.model.ParameterValueRest; import org.dspace.app.rest.model.ParameterValueRest;
import org.dspace.app.rest.model.ProcessRest; import org.dspace.app.rest.model.ProcessRest;
import org.dspace.app.rest.model.ScriptRest; import org.dspace.app.rest.model.ScriptRest;
@@ -40,6 +41,7 @@ import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Pageable;
import org.springframework.security.access.AccessDeniedException; import org.springframework.security.access.AccessDeniedException;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.web.multipart.MultipartFile;
/** /**
* This is the REST repository dealing with the Script logic * This is the REST repository dealing with the Script logic
@@ -103,7 +105,8 @@ public class ScriptRestRepository extends DSpaceRestRepository<ScriptRest, Strin
* @throws SQLException If something goes wrong * @throws SQLException If something goes wrong
* @throws IOException If something goes wrong * @throws IOException If something goes wrong
*/ */
public ProcessRest startProcess(String scriptName) throws SQLException, IOException, AuthorizeException { public ProcessRest startProcess(String scriptName,
List<MultipartFile> files) throws SQLException, IOException, AuthorizeException {
Context context = obtainContext(); Context context = obtainContext();
String properties = requestService.getCurrentRequest().getServletRequest().getParameter("properties"); String properties = requestService.getCurrentRequest().getServletRequest().getParameter("properties");
List<DSpaceCommandLineParameter> dSpaceCommandLineParameters = List<DSpaceCommandLineParameter> dSpaceCommandLineParameters =
@@ -119,7 +122,7 @@ public class ScriptRestRepository extends DSpaceRestRepository<ScriptRest, Strin
context.getCurrentUser(), scriptName, dSpaceCommandLineParameters); context.getCurrentUser(), scriptName, dSpaceCommandLineParameters);
List<String> args = constructArgs(dSpaceCommandLineParameters); List<String> args = constructArgs(dSpaceCommandLineParameters);
try { try {
runDSpaceScript(scriptToExecute, restDSpaceRunnableHandler, args); runDSpaceScript(files, context, scriptToExecute, restDSpaceRunnableHandler, args);
context.complete(); context.complete();
return processConverter.fromModel(restDSpaceRunnableHandler.getProcess()); return processConverter.fromModel(restDSpaceRunnableHandler.getProcess());
} catch (SQLException e) { } catch (SQLException e) {
@@ -167,10 +170,13 @@ public class ScriptRestRepository extends DSpaceRestRepository<ScriptRest, Strin
return args; return args;
} }
private void runDSpaceScript(DSpaceRunnable scriptToExecute, private void runDSpaceScript(List<MultipartFile> files, Context context, DSpaceRunnable scriptToExecute,
RestDSpaceRunnableHandler restDSpaceRunnableHandler, List<String> args) { RestDSpaceRunnableHandler restDSpaceRunnableHandler, List<String> args)
throws IOException {
try { try {
scriptToExecute.initialize(args.toArray(new String[0]), restDSpaceRunnableHandler); scriptToExecute.initialize(args.toArray(new String[0]), restDSpaceRunnableHandler);
checkFileNames(scriptToExecute, files);
processFiles(context, restDSpaceRunnableHandler, files);
restDSpaceRunnableHandler.schedule(scriptToExecute); restDSpaceRunnableHandler.schedule(scriptToExecute);
} catch (ParseException e) { } catch (ParseException e) {
scriptToExecute.printHelp(); scriptToExecute.printHelp();
@@ -181,4 +187,31 @@ public class ScriptRestRepository extends DSpaceRestRepository<ScriptRest, Strin
} }
} }
private void processFiles(Context context, RestDSpaceRunnableHandler restDSpaceRunnableHandler,
List<MultipartFile> files)
throws IOException {
for (MultipartFile file : files) {
restDSpaceRunnableHandler
.writeFilestream(context, file.getOriginalFilename(), file.getInputStream(), "inputfile");
}
}
private void checkFileNames(DSpaceRunnable scriptToExecute, List<MultipartFile> files) {
List<String> fileNames = new LinkedList<>();
for (MultipartFile file : files) {
String fileName = file.getOriginalFilename();
if (fileNames.contains(fileName)) {
throw new UnprocessableEntityException("There are two files with the same name: " + fileName);
} else {
fileNames.add(fileName);
}
}
List<String> fileNamesFromOptions = scriptToExecute.getFileNamesFromInputStreamOptions();
if (!fileNames.containsAll(fileNamesFromOptions)) {
throw new UnprocessableEntityException("Files given in properties aren't all present in the request");
}
}
} }

37
dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java
View File

@@ -7,6 +7,8 @@
*/ */
package org.dspace.app.rest.scripts.handler.impl; package org.dspace.app.rest.scripts.handler.impl;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter; import java.io.PrintWriter;
import java.io.StringWriter; import java.io.StringWriter;
import java.sql.SQLException; import java.sql.SQLException;
@@ -16,8 +18,12 @@ import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options; import org.apache.commons.cli.Options;
import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.dspace.authorize.AuthorizeException;
import org.dspace.content.Bitstream;
import org.dspace.content.ProcessStatus; import org.dspace.content.ProcessStatus;
import org.dspace.content.factory.ContentServiceFactory;
import org.dspace.content.factory.ProcessServiceFactory; import org.dspace.content.factory.ProcessServiceFactory;
import org.dspace.content.service.BitstreamService;
import org.dspace.core.Context; import org.dspace.core.Context;
import org.dspace.eperson.EPerson; import org.dspace.eperson.EPerson;
import org.dspace.scripts.DSpaceCommandLineParameter; import org.dspace.scripts.DSpaceCommandLineParameter;
@@ -25,6 +31,8 @@ import org.dspace.scripts.DSpaceRunnable;
import org.dspace.scripts.Process; import org.dspace.scripts.Process;
import org.dspace.scripts.handler.DSpaceRunnableHandler; import org.dspace.scripts.handler.DSpaceRunnableHandler;
import org.dspace.scripts.service.ProcessService; import org.dspace.scripts.service.ProcessService;
import org.dspace.utils.DSpace;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
/** /**
* The {@link DSpaceRunnableHandler} dealing with Scripts started from the REST api * The {@link DSpaceRunnableHandler} dealing with Scripts started from the REST api
@@ -34,6 +42,7 @@ public class RestDSpaceRunnableHandler implements DSpaceRunnableHandler {
.getLogger(RestDSpaceRunnableHandler.class); .getLogger(RestDSpaceRunnableHandler.class);
private ProcessService processService = ProcessServiceFactory.getInstance().getProcessService(); private ProcessService processService = ProcessServiceFactory.getInstance().getProcessService();
private BitstreamService bitstreamService = ContentServiceFactory.getInstance().getBitstreamService();
private Integer processId; private Integer processId;
private String scriptName; private String scriptName;
@@ -176,6 +185,29 @@ public class RestDSpaceRunnableHandler implements DSpaceRunnableHandler {
} }
} }
@Override
public InputStream getFileStream(Context context, String fileName) throws IOException, AuthorizeException {
try {
Process process = processService.find(context, processId);
Bitstream bitstream = processService.getBitstreamByName(context, process, fileName);
return bitstreamService.retrieve(context, bitstream);
} catch (SQLException sqlException) {
log.error("SQL exception while attempting to find process", sqlException);
}
return null;
}
@Override
public void writeFilestream(Context context, String fileName, InputStream inputStream, String type)
throws IOException {
try {
Process process = processService.find(context, processId);
processService.appendFile(context, process, inputStream, type, fileName);
} catch (SQLException | AuthorizeException exception) {
log.error("Exception occurred while attempting to find process", exception);
}
}
/** /**
* This method will return the process created by this handler * This method will return the process created by this handler
* @return The Process database object created by this handler * @return The Process database object created by this handler
@@ -200,6 +232,9 @@ public class RestDSpaceRunnableHandler implements DSpaceRunnableHandler {
* @param script The script to be ran * @param script The script to be ran
*/ */
public void schedule(DSpaceRunnable script) { public void schedule(DSpaceRunnable script) {
ThreadPoolTaskExecutor taskExecutor = new DSpace().getServiceManager()
.getServiceByName("dspaceRunnableThreadExecutor",
ThreadPoolTaskExecutor.class);
Context context = new Context(); Context context = new Context();
try { try {
Process process = processService.find(context, processId); Process process = processService.find(context, processId);
@@ -213,6 +248,6 @@ public class RestDSpaceRunnableHandler implements DSpaceRunnableHandler {
context.abort(); context.abort();
} }
} }
script.run(); taskExecutor.execute(script);
} }
} }

9
dspace/config/spring/rest/scripts.xml Normal file
View File

@@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">
<bean id="dspaceRunnableThreadExecutor" class="org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor">
<property name="corePoolSize" value="5"/>
</bean>
</beans>