hazza/DSpace
1
0
Fork 0
mirror of https://github.com/DSpace/DSpace.git synced 2025-10-07 01:54:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

[Task 66428] Applied feedback to the MetadataImport and Export scripts and framework

Browse Source
This commit is contained in:
Raf Ponsaerts
2019-12-10 16:13:54 +01:00
parent 84775128b1
commit e295a76c70
14 changed files with 142 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
dspace-api/src/main/java/org/dspace/app/bulkedit/DSpaceCSV.java
View File

@@ -8,12 +8,8 @@
package org.dspace.app.bulkedit;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
@@ -623,24 +619,6 @@ public class DSpaceCSV implements Serializable {
return csvLines;
}
/**
* Save the CSV file to the given filename
*
* @param filename The filename to save the CSV file to
* @throws IOException Thrown if an error occurs when writing the file
*/
public final void save(String filename) throws IOException {
// Save the file
BufferedWriter out = new BufferedWriter(
new OutputStreamWriter(
new FileOutputStream(filename), "UTF-8"));
for (String csvLine : getCSVLinesAsStringArray()) {
out.write(csvLine + "\n");
}
out.flush();
out.close();
}
public InputStream getInputStream() {
StringBuilder stringBuilder = new StringBuilder();
for (String csvLine : getCSVLinesAsStringArray()) {

29
dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExport.java
View File

@@ -21,7 +21,7 @@ import org.springframework.beans.factory.annotation.Autowired;
*/
public class MetadataExport extends DSpaceRunnable {
private Context c = null;
private Context context = null;
private boolean help = false;
private String filename = null;
private String handle = null;
@@ -31,43 +31,46 @@ public class MetadataExport extends DSpaceRunnable {
@Autowired
private MetadataExportService metadataExportService;
protected Context context;
private MetadataExport() {
Options options = constructOptions();
this.options = options;
this.options = constructOptions();
}
private Options constructOptions() {
Options options = new Options();
options.addOption("i", "id", true, "ID or handle of thing to export (item, collection, or community)");
options.getOption("i").setType(String.class);
options.addOption("f", "file", true, "destination where you want file written");
options.getOption("f").setType(String.class);
options.getOption("f").setRequired(true);
options.addOption("a", "all", false,
"include all metadata fields that are not normally changed (e.g. provenance)");
options.getOption("a").setType(boolean.class);
options.addOption("h", "help", false, "help");
options.getOption("h").setType(boolean.class);
return options;
}
public void internalRun() throws Exception {
if (help) {
handler.logInfo("\nfull export: metadataexport -f filename");
handler.logInfo("partial export: metadataexport -i handle -f filename");
handler.logInfo("\nfull export: metadata-export -f filename");
handler.logInfo("partial export: metadata-export -i handle -f filename");
printHelp();
return;
}
DSpaceCSV dSpaceCSV = metadataExportService.handleExport(c, exportAllItems, exportAllMetadata, handle);
handler.writeFilestream(c, filename, dSpaceCSV.getInputStream(), "exportCSV");
c.restoreAuthSystemState();
c.complete();
DSpaceCSV dSpaceCSV = metadataExportService.handleExport(context, exportAllItems, exportAllMetadata, handle,
handler);
handler.writeFilestream(context, filename, dSpaceCSV.getInputStream(), "exportCSV");
context.restoreAuthSystemState();
context.complete();
}
public void setup() throws ParseException {
c = new Context();
c.turnOffAuthorisationSystem();
context = new Context();
context.turnOffAuthorisationSystem();
if (commandLine.hasOption('h')) {
help = true;

24
dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImport.java
View File

@@ -21,7 +21,6 @@ import java.util.Set;
import java.util.UUID;
import javax.annotation.Nullable;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.lang3.StringUtils;
@@ -59,6 +58,7 @@ import org.dspace.eperson.EPerson;
import org.dspace.eperson.factory.EPersonServiceFactory;
import org.dspace.handle.service.HandleService;
import org.dspace.scripts.DSpaceRunnable;
import org.dspace.scripts.handler.DSpaceRunnableHandler;
import org.dspace.workflow.WorkflowItem;
import org.dspace.workflow.WorkflowService;
import org.dspace.workflow.factory.WorkflowServiceFactory;
@@ -219,7 +219,7 @@ public class MetadataImport extends DSpaceRunnable implements InitializingBean {
try {
// Ask the user if they want to make the changes
handler.logInfo("\n" + changeCounter + " item(s) will be changed\n");
change = handler.getUserValidation();
change = determineChange(handler);
} catch (IOException ioe) {
throw new IOException("Error: " + ioe.getMessage() + ", No changes have been made", ioe);
@@ -256,6 +256,10 @@ public class MetadataImport extends DSpaceRunnable implements InitializingBean {
}
protected boolean determineChange(DSpaceRunnableHandler handler) throws IOException {
return true;
}
public void setup() throws ParseException {
useTemplate = false;
filename = null;
@@ -322,7 +326,7 @@ public class MetadataImport extends DSpaceRunnable implements InitializingBean {
change = false;
}
private MetadataImport() {
public MetadataImport() {
Options options = constructOptions();
this.options = options;
}
@@ -1230,20 +1234,6 @@ public class MetadataImport extends DSpaceRunnable implements InitializingBean {
return in.replaceAll("\r\n", "").replaceAll("\n", "").trim();
}
/**
* Print the help message
*
* @param options The command line options the user gave
* @param exitCode the system exit code to use
*/
private static void printHelp(Options options, int exitCode) {
// print the help message
HelpFormatter myhelp = new HelpFormatter();
myhelp.printHelp("MetatadataImport\n", options);
System.out.println("\nmetadataimport: MetadataImport -f filename");
System.exit(exitCode);
}
/**
* Display the changes that have been detected, or that have been made
*

29
dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImportCLI.java Normal file
View File

@@ -0,0 +1,29 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.bulkedit;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import org.dspace.scripts.handler.DSpaceRunnableHandler;
public class MetadataImportCLI extends MetadataImport {
@Override
protected boolean determineChange(DSpaceRunnableHandler handler) throws IOException {
handler.logInfo("Do you want to make these changes? [y/n] ");
try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(System.in))) {
String yn = bufferedReader.readLine();
if ("y".equalsIgnoreCase(yn)) {
return true;
}
return false;
}
}
}

24
dspace-api/src/main/java/org/dspace/content/MetadataExportServiceImpl.java
View File

@@ -13,28 +13,27 @@ import java.util.Iterator;
import java.util.List;
import com.google.common.collect.Iterators;
import org.apache.logging.log4j.Logger;
import org.dspace.app.bulkedit.DSpaceCSV;
import org.dspace.content.service.ItemService;
import org.dspace.content.service.MetadataExportService;
import org.dspace.core.Constants;
import org.dspace.core.Context;
import org.dspace.handle.factory.HandleServiceFactory;
import org.dspace.scripts.handler.DSpaceRunnableHandler;
import org.springframework.beans.factory.annotation.Autowired;
public class MetadataExportServiceImpl implements MetadataExportService {
private static final Logger log = org.apache.logging.log4j.LogManager.getLogger(MetadataExportServiceImpl.class);
@Autowired
private ItemService itemService;
public DSpaceCSV handleExport(Context context, boolean exportAllItems, boolean exportAllMetadata, String handle)
throws Exception {
@Override
public DSpaceCSV handleExport(Context context, boolean exportAllItems, boolean exportAllMetadata, String handle,
DSpaceRunnableHandler handler) throws Exception {
Iterator<Item> toExport = null;
if (!exportAllItems) {
log.info("Exporting whole repository WARNING: May take some time!");
handler.logInfo("Exporting whole repository WARNING: May take some time!");
toExport = itemService.findAll(context);
} else {
DSpaceObject dso = HandleServiceFactory.getInstance().getHandleService().resolveToObject(context, handle);
@@ -44,16 +43,16 @@ public class MetadataExportServiceImpl implements MetadataExportService {
}
if (dso.getType() == Constants.ITEM) {
log.info("Exporting item '" + dso.getName() + "' (" + handle + ")");
handler.logInfo("Exporting item '" + dso.getName() + "' (" + handle + ")");
List<Item> item = new ArrayList<>();
item.add((Item) dso);
toExport = item.iterator();
} else if (dso.getType() == Constants.COLLECTION) {
log.info("Exporting collection '" + dso.getName() + "' (" + handle + ")");
handler.logInfo("Exporting collection '" + dso.getName() + "' (" + handle + ")");
Collection collection = (Collection) dso;
toExport = itemService.findByCollection(context, collection);
} else if (dso.getType() == Constants.COMMUNITY) {
log.info("Exporting community '" + dso.getName() + "' (" + handle + ")");
handler.logInfo("Exporting community '" + dso.getName() + "' (" + handle + ")");
toExport = buildFromCommunity(context, (Community) dso);
} else {
throw new IllegalArgumentException("Error identifying '" + handle + "'");
@@ -64,11 +63,7 @@ public class MetadataExportServiceImpl implements MetadataExportService {
return csv;
}
/**
* Run the export
*
* @return the exported CSV lines
*/
@Override
public DSpaceCSV export(Context context, Iterator<Item> toExport, boolean exportAll) throws Exception {
Context.Mode originalMode = context.getCurrentMode();
context.setMode(Context.Mode.READ_ONLY);
@@ -86,6 +81,7 @@ public class MetadataExportServiceImpl implements MetadataExportService {
return csv;
}
@Override
public DSpaceCSV export(Context context, Community community, boolean exportAll) throws Exception {
return export(context, buildFromCommunity(context, community), exportAll);
}

36
dspace-api/src/main/java/org/dspace/content/service/MetadataExportService.java
View File

@@ -13,14 +13,46 @@ import org.dspace.app.bulkedit.DSpaceCSV;
import org.dspace.content.Community;
import org.dspace.content.Item;
import org.dspace.core.Context;
import org.dspace.scripts.handler.DSpaceRunnableHandler;
/**
* This is the interface to be implemented by a Service that deals with the exporting of Metadata
*/
public interface MetadataExportService {
public DSpaceCSV handleExport(Context context, boolean exportAllItems, boolean exportAllMetadata, String handle)
throws Exception;
/**
* This method will export DSpaceObject objects depending on the parameters it gets. It can export all the items
* in the repository, all the items in a community, all the items in a collection or a specific item. The latter
* three are specified by the handle parameter. The entire repository can be exported by defining the
* exportAllItems parameter as true
* @param context The relevant DSpace context
* @param exportAllItems A boolean indicating whether or not the entire repository should be exported
* @param exportAllMetadata Defines if all metadata should be exported or only the allowed ones
* @param handle The handle for the DSpaceObject to be exported, can be a Community, Collection or Item
* @return A DSpaceCSV object containing the exported information
* @throws Exception If something goes wrong
*/
public DSpaceCSV handleExport(Context context, boolean exportAllItems, boolean exportAllMetadata,
String handle, DSpaceRunnableHandler dSpaceRunnableHandler) throws Exception;
/**
* This method will export all the Items in the given toExport iterator to a DSpaceCSV
* @param context The relevant DSpace context
* @param toExport The iterator containing the items to export
* @param exportAll Defines if all metadata should be exported or only the allowed ones
* @return A DSpaceCSV object containing the exported information
* @throws Exception If something goes wrong
*/
public DSpaceCSV export(Context context, Iterator<Item> toExport, boolean exportAll) throws Exception;
/**
* This method will export all the Items within the given Community to a DSpaceCSV
* @param context The relevant DSpace context
* @param community The Community that contains the Items to be exported
* @param exportAll Defines if all metadata should be exported or only the allowed ones
* @return A DSpaceCSV object containing the exported information
* @throws Exception If something goes wrong
*/
public DSpaceCSV export(Context context, Community community, boolean exportAll) throws Exception;
}

5
dspace-api/src/main/java/org/dspace/scripts/DSpaceRunnable.java
View File

@@ -77,6 +77,11 @@ public abstract class DSpaceRunnable implements Runnable {
return options;
}
/**
* This method will traverse all the options and it'll grab options defined as an InputStream type to then save
* the filename specified by that option in a list of Strings that'll be returned in the end
* @return The list of Strings representing filenames from the options given to the script
*/
public List<String> getFileNamesFromInputStreamOptions() {
List<String> fileNames = new LinkedList<>();

1
dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java
View File

@@ -184,6 +184,7 @@ public class ProcessServiceImpl implements ProcessService {
return null;
}
@Override
public List<Bitstream> getBitstreams(Context context, Process process, String type) {
List<Bitstream> allBitstreams = process.getBitstreams();

22
dspace-api/src/main/java/org/dspace/scripts/handler/DSpaceRunnableHandler.java
View File

@@ -83,11 +83,27 @@ public interface DSpaceRunnableHandler {
*/
public void printHelp(Options options, String name);
/**
* This method will grab the InputStream for the file defined by the given file name. The exact implementation will
* differ based on whether it's a REST call or CommandLine call. The REST Call will look for Bitstreams in the
* Database whereas the CommandLine call will look on the filesystem
* @param context The relevant DSpace context
* @param fileName The filename for the file that holds the InputStream
* @return The InputStream for the file defined by the given file name
* @throws IOException If something goes wrong
* @throws AuthorizeException If something goes wrong
*/
public InputStream getFileStream(Context context, String fileName) throws IOException, AuthorizeException;
/**
* This method will write the InputStream to either a file on the filesystem or a bitstream in the database
* depending on whether it's coming from a CommandLine call or REST call respectively
* @param context The relevant DSpace context
* @param fileName The filename
* @param inputStream The inputstream to be written
* @param type The type of the file
* @throws IOException If something goes wrong
*/
public void writeFilestream(Context context, String fileName, InputStream inputStream, String type)
throws IOException;
boolean getUserValidation() throws IOException;
}

12
dspace-api/src/main/java/org/dspace/scripts/handler/impl/CommandLineDSpaceRunnableHandler.java
View File

@@ -7,11 +7,9 @@
*/
package org.dspace.scripts.handler.impl;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
@@ -108,14 +106,4 @@ public class CommandLineDSpaceRunnableHandler implements DSpaceRunnableHandler {
File file = new File(fileName);
FileUtils.copyInputStreamToFile(inputStream, file);
}
@Override
public boolean getUserValidation() throws IOException {
logInfo("Do you want to make these changes? [y/n] ");
String yn = (new BufferedReader(new InputStreamReader(System.in))).readLine();
if ("y".equalsIgnoreCase(yn)) {
return true;
}
return false;
}
}

16
dspace-api/src/main/java/org/dspace/scripts/service/ProcessService.java
View File

@@ -135,8 +135,24 @@ public interface ProcessService {
*/
public List<DSpaceCommandLineParameter> getParameters(Process process);
/**
* This method will return the Bitstream that matches the given name for the given Process
* @param context The relevant DSpace context
* @param process The process that should hold the requested Bitstream
* @param bitstreamName The name of the requested Bitstream
* @return The Bitstream from the given Process that matches the given bitstream name
*/
public Bitstream getBitstreamByName(Context context, Process process, String bitstreamName);
/**
* This method will return all the Bitstreams for a given process if the type is defined as null. If type is
* different than null, the bitstreams with metadata process.type equal to the given type from that process
* are returned
* @param context The relevant DSpace context
* @param process The process that holds the Bitstreams to be searched in
* @param type The type that the Bitstream must have
* @return The list of Bitstreams of the given type for the given Process
*/
public List<Bitstream> getBitstreams(Context context, Process process, String type);
/**

5
dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java
View File

@@ -250,9 +250,4 @@ public class RestDSpaceRunnableHandler implements DSpaceRunnableHandler {
}
taskExecutor.execute(script);
}
@Override
public boolean getUserValidation() throws IOException {
return true;
}
}

2
dspace/config/spring/api/scripts.xml
View File

@@ -9,7 +9,7 @@
<property name="description" value="Update Discovery Solr Search Index"/>
</bean>
<bean id="MetadataImport" class="org.dspace.app.bulkedit.MetadataImport" scope="prototype">
<bean id="MetadataImportCLI" class="org.dspace.app.bulkedit.MetadataImportCLI" scope="prototype">
<property name="name" value="metadata-import" />
<property name="description" value="Import metadata after batch editing" />
</bean>

4
dspace/config/spring/rest/scripts.xml
View File

@@ -6,4 +6,8 @@
<property name="corePoolSize" value="5"/>
</bean>
<bean id="MetadataImport" class="org.dspace.app.bulkedit.MetadataImport" scope="prototype">
<property name="name" value="metadata-import" />
<property name="description" value="Import metadata after batch editing" />
</bean>
</beans>