hazza/DSpace
1
0
Fork 0
mirror of https://github.com/DSpace/DSpace.git synced 2025-10-19 07:53:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

[Task 64926] applied the feedback

Browse Source
This commit is contained in:
Raf Ponsaerts
2019-09-16 09:01:41 +02:00
parent 6e95aae040
commit 9675df2d68
22 changed files with 414 additions and 516 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
dspace-api/src/main/java/org/dspace/content/ProcessServiceImpl.java
View File

@@ -32,9 +32,6 @@ public class ProcessServiceImpl implements ProcessService {
@Autowired
private ProcessDAO processDAO;
@Autowired
private BitstreamService bitstreamService;
@Override
public Process create(Context context, EPerson ePerson, String scriptName,
List<DSpaceCommandLineParameter> parameters) throws SQLException {
@@ -60,6 +57,11 @@ public class ProcessServiceImpl implements ProcessService {
return processDAO.findAll(context, Process.class);
}
@Override
public List<Process> findAll(Context context, int limit, int offset) throws SQLException {
return processDAO.findAll(context, limit, offset);
}
@Override
public List<Process> findAllSortByScript(Context context) throws SQLException {
return processDAO.findAllSortByScript(context);
@@ -130,4 +132,8 @@ public class ProcessServiceImpl implements ProcessService {
return parameterList;
}
public int countTotal(Context context) throws SQLException {
return processDAO.countRows(context);
}
}

13
dspace-api/src/main/java/org/dspace/content/dao/ProcessDAO.java
View File

@@ -31,4 +31,17 @@ public interface ProcessDAO extends GenericDAO<Process> {
* @throws SQLException If something goes wrong
*/
public List<Process> findAllSortByStartTime(Context context) throws SQLException;
/**
* Returns a list of all Process objects in the database
* @param context The relevant DSpace context
* @param limit The limit for the amount of Processes returned
* @param offset The offset for the Processes to be returned
* @return The list of all Process objects in the Database
* @throws SQLException If something goes wrong
*/
List<Process> findAll(Context context, int limit, int offset) throws SQLException;
int countRows(Context context) throws SQLException;
}

20
dspace-api/src/main/java/org/dspace/content/dao/impl/ProcessDAOImpl.java
View File

@@ -45,6 +45,26 @@ public class ProcessDAOImpl extends AbstractHibernateDAO<Process> implements Pro
return list(context, criteriaQuery, false, Process.class, -1, -1);
}
public List<Process> findAll(Context context, int limit, int offset) throws SQLException {
CriteriaBuilder criteriaBuilder = getCriteriaBuilder(context);
CriteriaQuery criteriaQuery = getCriteriaQuery(criteriaBuilder, Process.class);
Root<Process> processRoot = criteriaQuery.from(Process.class);
criteriaQuery.select(processRoot);
return list(context, criteriaQuery, false, Process.class, limit, offset);
}
public int countRows(Context context) throws SQLException {
CriteriaBuilder criteriaBuilder = getCriteriaBuilder(context);
CriteriaQuery criteriaQuery = getCriteriaQuery(criteriaBuilder, Process.class);
Root<Process> processRoot = criteriaQuery.from(Process.class);
criteriaQuery.select(processRoot);
return count(context, criteriaQuery, criteriaBuilder, processRoot);
}
}

14
dspace-api/src/main/java/org/dspace/content/service/ProcessService.java
View File

@@ -49,6 +49,17 @@ public interface ProcessService {
*/
public List<Process> findAll(Context context) throws SQLException;
/**
* Returns a list of all Process objects in the database
* @param context The relevant DSpace context
* @param limit The limit for the amount of Processes returned
* @param offset The offset for the Processes to be returned
* @return The list of all Process objects in the Database
* @throws SQLException If something goes wrong
*/
public List<Process> findAll(Context context, int limit, int offset) throws SQLException;
/**
* Returns a list of all Process objects in the database sorted by script name
* @param context The relevant DSpace context
@@ -115,4 +126,7 @@ public interface ProcessService {
* @return The list of parsed parameters from the Process object
*/
public List<DSpaceCommandLineParameter> getParameters(Process process);
int countTotal(Context context) throws SQLException;
}

229
dspace-api/src/main/java/org/dspace/discovery/IndexClient.java
View File

@@ -13,11 +13,8 @@ import java.util.Iterator;
import java.util.UUID;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.PosixParser;
import org.apache.logging.log4j.Logger;
import org.apache.commons.cli.ParseException;
import org.dspace.content.Collection;
import org.dspace.content.Community;
import org.dspace.content.Item;
@@ -26,103 +23,23 @@ import org.dspace.content.service.ItemService;
import org.dspace.core.Constants;
import org.dspace.core.Context;
import org.dspace.handle.factory.HandleServiceFactory;
import org.dspace.services.factory.DSpaceServicesFactory;
import org.dspace.scripts.DSpaceRunnable;
import org.springframework.beans.factory.annotation.Autowired;
/**
* Class used to reindex dspace communities/collections/items into discovery
*
* @author Kevin Van de Velde (kevin at atmire dot com)
* @author Mark Diggory (markd at atmire dot com)
* @author Ben Bosman (ben at atmire dot com)
*/
public class IndexClient {
public class IndexClient extends DSpaceRunnable {
private static final Logger log = org.apache.logging.log4j.LogManager.getLogger(IndexClient.class);
private Context context;
/**
* Default constructor
*/
private IndexClient() { }
@Autowired
private IndexingService indexer;
/**
* When invoked as a command-line tool, creates, updates, removes content
* from the whole index
*
* @param args the command-line arguments, none used
* @throws SQLException An exception that provides information on a database access error or other errors.
* @throws IOException A general class of exceptions produced by failed or interrupted I/O operations.
* @throws SearchServiceException if something went wrong with querying the solr server
*/
public static void main(String[] args) throws SQLException, IOException, SearchServiceException {
private IndexClientOptions indexClientOptions;
Context context = new Context(Context.Mode.READ_ONLY);
context.turnOffAuthorisationSystem();
String usage = "org.dspace.discovery.IndexClient [-cbhf] | [-r <handle>] | [-i <handle>] or nothing to " +
"update/clean an existing index.";
Options options = new Options();
HelpFormatter formatter = new HelpFormatter();
CommandLine line = null;
options.addOption(OptionBuilder
.withArgName("handle to remove")
.hasArg(true)
.withDescription(
"remove an Item, Collection or Community from index based on its handle")
.create("r"));
options.addOption(OptionBuilder
.withArgName("handle or uuid to add or update")
.hasArg(true)
.withDescription(
"add or update an Item, Collection or Community based on its handle or uuid")
.create("i"));
options.addOption(OptionBuilder
.isRequired(false)
.withDescription(
"clean existing index removing any documents that no longer exist in the db")
.create("c"));
options.addOption(OptionBuilder
.isRequired(false)
.withDescription(
"(re)build index, wiping out current one if it exists")
.create("b"));
options.addOption(OptionBuilder
.isRequired(false)
.withDescription(
"Rebuild the spellchecker, can be combined with -b and -f.")
.create("s"));
options.addOption(OptionBuilder
.isRequired(false)
.withDescription(
"if updating existing index, force each handle to be reindexed even if uptodate")
.create("f"));
options.addOption(OptionBuilder
.isRequired(false)
.withDescription(
"print this help message")
.create("h"));
options.addOption(OptionBuilder.isRequired(false).withDescription(
"optimize search core").create("o"));
try {
line = new PosixParser().parse(options, args);
} catch (Exception e) {
// automatically generate the help statement
formatter.printHelp(usage, e.getMessage(), options, "");
System.exit(1);
}
if (line.hasOption("h")) {
// automatically generate the help statement
formatter.printHelp(usage, options);
System.exit(1);
@Override
public void internalRun() throws Exception {
if (indexClientOptions == IndexClientOptions.HELP) {
printHelp();
return;
}
/** Acquire from dspace-services in future */
@@ -130,28 +47,29 @@ public class IndexClient {
* new DSpace.getServiceManager().getServiceByName("org.dspace.discovery.SolrIndexer");
*/
IndexingService indexer = DSpaceServicesFactory.getInstance().getServiceManager().getServiceByName(
IndexingService.class.getName(),
IndexingService.class
);
if (line.hasOption("r")) {
log.info("Removing " + line.getOptionValue("r") + " from Index");
indexer.unIndexContent(context, line.getOptionValue("r"));
} else if (line.hasOption("c")) {
log.info("Cleaning Index");
indexer.cleanIndex(line.hasOption("f"));
} else if (line.hasOption("b")) {
log.info("(Re)building index from scratch.");
if (indexClientOptions == IndexClientOptions.REMOVE) {
handler.logInfo("Removing " + commandLine.getOptionValue("r") + " from Index");
indexer.unIndexContent(context, commandLine.getOptionValue("r"));
} else if (indexClientOptions == IndexClientOptions.CLEAN) {
handler.logInfo("Cleaning Index");
indexer.cleanIndex(false);
} else if (indexClientOptions == IndexClientOptions.FORCECLEAN) {
handler.logInfo("Cleaning Index");
indexer.cleanIndex(true);
} else if (indexClientOptions == IndexClientOptions.BUILD ||
indexClientOptions == IndexClientOptions.BUILDANDSPELLCHECK) {
handler.logInfo("(Re)building index from scratch.");
indexer.createIndex(context);
checkRebuildSpellCheck(line, indexer);
} else if (line.hasOption("o")) {
log.info("Optimizing search core.");
if (indexClientOptions == IndexClientOptions.BUILDANDSPELLCHECK) {
checkRebuildSpellCheck(commandLine, indexer);
}
} else if (indexClientOptions == IndexClientOptions.OPTIMIZE) {
handler.logInfo("Optimizing search core.");
indexer.optimize();
} else if (line.hasOption('s')) {
checkRebuildSpellCheck(line, indexer);
} else if (line.hasOption('i')) {
final String param = line.getOptionValue('i');
} else if (indexClientOptions == IndexClientOptions.SPELLCHECK) {
checkRebuildSpellCheck(commandLine, indexer);
} else if (indexClientOptions == IndexClientOptions.INDEX) {
final String param = commandLine.getOptionValue('i');
UUID uuid = null;
try {
uuid = UUID.fromString(param);
@@ -176,19 +94,71 @@ public class IndexClient {
if (dso == null) {
throw new IllegalArgumentException("Cannot resolve " + param + " to a DSpace object");
}
log.info("Indexing " + param + " force " + line.hasOption("f"));
handler.logInfo("Indexing " + param + " force " + commandLine.hasOption("f"));
final long startTimeMillis = System.currentTimeMillis();
final long count = indexAll(indexer, ContentServiceFactory.getInstance().getItemService(), context, dso);
final long count = indexAll(indexer, ContentServiceFactory.getInstance().getItemService(), context,
dso);
final long seconds = (System.currentTimeMillis() - startTimeMillis) / 1000;
log.info("Indexed " + count + " object" + (count > 1 ? "s" : "") + " in " + seconds + " seconds");
} else {
log.info("Updating and Cleaning Index");
indexer.cleanIndex(line.hasOption("f"));
indexer.updateIndex(context, line.hasOption("f"));
checkRebuildSpellCheck(line, indexer);
handler.logInfo("Indexed " + count + " object" + (count > 1 ? "s" : "") + " in " + seconds + " seconds");
} else if (indexClientOptions == IndexClientOptions.UPDATE ||
indexClientOptions == IndexClientOptions.UPDATEANDSPELLCHECK) {
handler.logInfo("Updating and Cleaning Index");
indexer.cleanIndex(false);
indexer.updateIndex(context, false);
if (indexClientOptions == IndexClientOptions.UPDATEANDSPELLCHECK) {
checkRebuildSpellCheck(commandLine, indexer);
}
} else if (indexClientOptions == IndexClientOptions.FORCEUPDATE ||
indexClientOptions == IndexClientOptions.FORCEUPDATEANDSPELLCHECK) {
handler.logInfo("Updating and Cleaning Index");
indexer.cleanIndex(true);
indexer.updateIndex(context, true);
if (indexClientOptions == IndexClientOptions.FORCEUPDATEANDSPELLCHECK) {
checkRebuildSpellCheck(commandLine, indexer);
}
}
log.info("Done with indexing");
handler.logInfo("Done with indexing");
}
public void setup() throws ParseException {
try {
context = new Context(Context.Mode.READ_ONLY);
context.turnOffAuthorisationSystem();
} catch (Exception e) {
throw new ParseException("Unable to create a new DSpace Context: " + e.getMessage());
}
indexClientOptions = IndexClientOptions.getIndexClientOption(commandLine);
}
private IndexClient() {
Options options = constructOptions();
this.options = options;
}
private Options constructOptions() {
Options options = new Options();
options
.addOption("r", "remove", true, "remove an Item, Collection or Community from index based on its handle");
options.getOption("r").setType(String.class);
options.addOption("i", "index", true,
"add or update an Item, Collection or Community based on its handle or uuid");
options.getOption("i").setType(boolean.class);
options.addOption("c", "clean", false,
"clean existing index removing any documents that no longer exist in the db");
options.getOption("c").setType(boolean.class);
options.addOption("b", "build", false, "(re)build index, wiping out current one if it exists");
options.getOption("b").setType(boolean.class);
options.addOption("s", "spellchecker", false, "Rebuild the spellchecker, can be combined with -b and -f.");
options.getOption("s").setType(boolean.class);
options.addOption("f", "force", false,
"if updating existing index, force each handle to be reindexed even if uptodate");
options.getOption("f").setType(boolean.class);
options.addOption("h", "help", false, "print this help message");
options.getOption("h").setType(boolean.class);
return options;
}
/**
@@ -273,13 +243,12 @@ public class IndexClient {
* @param line the command line options
* @param indexer the solr indexer
* @throws SearchServiceException in case of a solr exception
* @throws java.io.IOException passed through
* @throws IOException passed through
*/
protected static void checkRebuildSpellCheck(CommandLine line, IndexingService indexer)
protected void checkRebuildSpellCheck(CommandLine line, IndexingService indexer)
throws SearchServiceException, IOException {
if (line.hasOption("s")) {
log.info("Rebuilding spell checker.");
handler.logInfo("Rebuilding spell checker.");
indexer.buildSpellCheck();
}
}
}

2
dspace-api/src/main/java/org/dspace/scripts/IndexClientOptions.java → dspace-api/src/main/java/org/dspace/discovery/IndexClientOptions.java
View File

@@ -6,7 +6,7 @@
* http://www.dspace.org/license/
*/
package org.dspace.scripts;
package org.dspace.discovery;
import org.apache.commons.cli.CommandLine;

3
dspace-api/src/main/java/org/dspace/handle/UpdateHandlePrefix.java
View File

@@ -13,6 +13,7 @@ import java.sql.SQLException;
import java.util.Iterator;
import org.apache.logging.log4j.Logger;
import org.dspace.app.launcher.ScriptLauncher;
import org.dspace.content.MetadataValue;
import org.dspace.content.factory.ContentServiceFactory;
import org.dspace.content.service.MetadataValueService;
@@ -139,7 +140,7 @@ public class UpdateHandlePrefix {
try {
// Reinitialise the search and browse system
IndexClient.main(new String[] {"-b"});
ScriptLauncher.main(new String[] {"index-discovery", "-b"});
System.out.println("Browse and search indexes are ready now.");
// All done
System.out.println("\nAll done successfully. Please check the DSpace logs!\n");

17
dspace-api/src/main/java/org/dspace/scripts/DSpaceRunnable.java
View File

@@ -7,11 +7,16 @@
*/
package org.dspace.scripts;
import java.sql.SQLException;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.dspace.authorize.service.AuthorizeService;
import org.dspace.core.Context;
import org.dspace.scripts.handler.DSpaceRunnableHandler;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Required;
/**
@@ -42,6 +47,9 @@ public abstract class DSpaceRunnable implements Runnable {
*/
protected DSpaceRunnableHandler handler;
@Autowired
private AuthorizeService authorizeService;
public String getName() {
return name;
}
@@ -128,4 +136,13 @@ public abstract class DSpaceRunnable implements Runnable {
* @throws ParseException If something goes wrong
*/
public abstract void setup() throws ParseException;
public boolean isAllowedToExecute(Context context) {
try {
return authorizeService.isAdmin(context);
} catch (SQLException e) {
handler.logError("Error occured when trying to verify permissions for script: " + name);
}
return false;
}
}

258
dspace-api/src/main/java/org/dspace/scripts/impl/IndexClient.java
View File

@@ -1,258 +0,0 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.scripts.impl;
import java.io.IOException;
import java.sql.SQLException;
import java.util.Iterator;
import java.util.UUID;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.dspace.content.Collection;
import org.dspace.content.Community;
import org.dspace.content.Item;
import org.dspace.content.factory.ContentServiceFactory;
import org.dspace.content.service.ItemService;
import org.dspace.core.Constants;
import org.dspace.core.Context;
import org.dspace.discovery.IndexableObject;
import org.dspace.discovery.IndexingService;
import org.dspace.discovery.SearchServiceException;
import org.dspace.handle.factory.HandleServiceFactory;
import org.dspace.scripts.DSpaceRunnable;
import org.dspace.scripts.IndexClientOptions;
import org.springframework.beans.factory.annotation.Autowired;
public class IndexClient extends DSpaceRunnable {
private Context context;
@Autowired
private IndexingService indexer;
private IndexClientOptions indexClientOptions;
@Override
public void internalRun() throws Exception {
if (indexClientOptions == IndexClientOptions.HELP) {
printHelp();
return;
}
/** Acquire from dspace-services in future */
/**
* new DSpace.getServiceManager().getServiceByName("org.dspace.discovery.SolrIndexer");
*/
if (indexClientOptions == IndexClientOptions.REMOVE) {
handler.logInfo("Removing " + commandLine.getOptionValue("r") + " from Index");
indexer.unIndexContent(context, commandLine.getOptionValue("r"));
} else if (indexClientOptions == IndexClientOptions.CLEAN) {
handler.logInfo("Cleaning Index");
indexer.cleanIndex(false);
} else if (indexClientOptions == IndexClientOptions.FORCECLEAN) {
handler.logInfo("Cleaning Index");
indexer.cleanIndex(true);
} else if (indexClientOptions == IndexClientOptions.BUILD ||
indexClientOptions == IndexClientOptions.BUILDANDSPELLCHECK) {
handler.logInfo("(Re)building index from scratch.");
indexer.createIndex(context);
if (indexClientOptions == IndexClientOptions.BUILDANDSPELLCHECK) {
checkRebuildSpellCheck(commandLine, indexer);
}
} else if (indexClientOptions == IndexClientOptions.OPTIMIZE) {
handler.logInfo("Optimizing search core.");
indexer.optimize();
} else if (indexClientOptions == IndexClientOptions.SPELLCHECK) {
checkRebuildSpellCheck(commandLine, indexer);
} else if (indexClientOptions == IndexClientOptions.INDEX) {
final String param = commandLine.getOptionValue('i');
UUID uuid = null;
try {
uuid = UUID.fromString(param);
} catch (Exception e) {
// nothing to do, it should be an handle
}
IndexableObject dso = null;
if (uuid != null) {
dso = ContentServiceFactory.getInstance().getItemService().find(context, uuid);
if (dso == null) {
// it could be a community
dso = ContentServiceFactory.getInstance().getCommunityService().find(context, uuid);
if (dso == null) {
// it could be a collection
dso = ContentServiceFactory.getInstance().getCollectionService().find(context, uuid);
}
}
} else {
dso = (IndexableObject) HandleServiceFactory.getInstance()
.getHandleService().resolveToObject(context, param);
}
if (dso == null) {
throw new IllegalArgumentException("Cannot resolve " + param + " to a DSpace object");
}
handler.logInfo("Indexing " + param + " force " + commandLine.hasOption("f"));
final long startTimeMillis = System.currentTimeMillis();
final long count = indexAll(indexer, ContentServiceFactory.getInstance().getItemService(), context,
dso);
final long seconds = (System.currentTimeMillis() - startTimeMillis) / 1000;
handler.logInfo("Indexed " + count + " object" + (count > 1 ? "s" : "") + " in " + seconds + " seconds");
} else if (indexClientOptions == IndexClientOptions.UPDATE ||
indexClientOptions == IndexClientOptions.UPDATEANDSPELLCHECK) {
handler.logInfo("Updating and Cleaning Index");
indexer.cleanIndex(false);
indexer.updateIndex(context, false);
if (indexClientOptions == IndexClientOptions.UPDATEANDSPELLCHECK) {
checkRebuildSpellCheck(commandLine, indexer);
}
} else if (indexClientOptions == IndexClientOptions.FORCEUPDATE ||
indexClientOptions == IndexClientOptions.FORCEUPDATEANDSPELLCHECK) {
handler.logInfo("Updating and Cleaning Index");
indexer.cleanIndex(true);
indexer.updateIndex(context, true);
if (indexClientOptions == IndexClientOptions.FORCEUPDATEANDSPELLCHECK) {
checkRebuildSpellCheck(commandLine, indexer);
}
}
handler.logInfo("Done with indexing");
}
public void setup() throws ParseException {
try {
context = new Context(Context.Mode.READ_ONLY);
context.turnOffAuthorisationSystem();
} catch (Exception e) {
throw new ParseException("Unable to create a new DSpace Context: " + e.getMessage());
}
indexClientOptions = IndexClientOptions.getIndexClientOption(commandLine);
}
private IndexClient() {
Options options = constructOptions();
this.options = options;
}
private Options constructOptions() {
Options options = new Options();
options
.addOption("r", "remove", true, "remove an Item, Collection or Community from index based on its handle");
options.getOption("r").setType(String.class);
options.addOption("i", "index", true,
"add or update an Item, Collection or Community based on its handle or uuid");
options.getOption("i").setType(boolean.class);
options.addOption("c", "clean", false,
"clean existing index removing any documents that no longer exist in the db");
options.getOption("c").setType(boolean.class);
options.addOption("b", "build", false, "(re)build index, wiping out current one if it exists");
options.getOption("b").setType(boolean.class);
options.addOption("s", "spellchecker", false, "Rebuild the spellchecker, can be combined with -b and -f.");
options.getOption("s").setType(boolean.class);
options.addOption("f", "force", false,
"if updating existing index, force each handle to be reindexed even if uptodate");
options.getOption("f").setType(boolean.class);
options.addOption("h", "help", false, "print this help message");
options.getOption("h").setType(boolean.class);
return options;
}
/**
* Indexes the given object and all children, if applicable.
*
* @param indexingService
* @param itemService
* @param context The relevant DSpace Context.
* @param dso DSpace object to index recursively
* @throws IOException A general class of exceptions produced by failed or interrupted I/O operations.
* @throws SearchServiceException in case of a solr exception
* @throws SQLException An exception that provides information on a database access error or other errors.
*/
private static long indexAll(final IndexingService indexingService,
final ItemService itemService,
final Context context,
final IndexableObject dso)
throws IOException, SearchServiceException, SQLException {
long count = 0;
indexingService.indexContent(context, dso, true, true);
count++;
if (dso.getType() == Constants.COMMUNITY) {
final Community community = (Community) dso;
final String communityHandle = community.getHandle();
for (final Community subcommunity : community.getSubcommunities()) {
count += indexAll(indexingService, itemService, context, subcommunity);
//To prevent memory issues, discard an object from the cache after processing
context.uncacheEntity(subcommunity);
}
final Community reloadedCommunity = (Community) HandleServiceFactory.getInstance().getHandleService()
.resolveToObject(context,
communityHandle);
for (final Collection collection : reloadedCommunity.getCollections()) {
count++;
indexingService.indexContent(context, collection, true, true);
count += indexItems(indexingService, itemService, context, collection);
//To prevent memory issues, discard an object from the cache after processing
context.uncacheEntity(collection);
}
} else if (dso.getType() == Constants.COLLECTION) {
count += indexItems(indexingService, itemService, context, (Collection) dso);
}
return count;
}
/**
* Indexes all items in the given collection.
*
* @param indexingService
* @param itemService
* @param context The relevant DSpace Context.
* @param collection collection to index
* @throws IOException A general class of exceptions produced by failed or interrupted I/O operations.
* @throws SearchServiceException in case of a solr exception
* @throws SQLException An exception that provides information on a database access error or other errors.
*/
private static long indexItems(final IndexingService indexingService,
final ItemService itemService,
final Context context,
final Collection collection)
throws IOException, SearchServiceException, SQLException {
long count = 0;
final Iterator<Item> itemIterator = itemService.findByCollection(context, collection);
while (itemIterator.hasNext()) {
Item item = itemIterator.next();
indexingService.indexContent(context, item, true, false);
count++;
//To prevent memory issues, discard an object from the cache after processing
context.uncacheEntity(item);
}
indexingService.commit();
return count;
}
/**
* Check the command line options and rebuild the spell check if active.
*
* @param line the command line options
* @param indexer the solr indexer
* @throws SearchServiceException in case of a solr exception
* @throws IOException passed through
*/
protected void checkRebuildSpellCheck(CommandLine line, IndexingService indexer)
throws SearchServiceException, IOException {
handler.logInfo("Rebuilding spell checker.");
indexer.buildSpellCheck();
}
}

40
dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V7.0_2019_06_14__scripts-and-process.sql Normal file
View File

@@ -0,0 +1,40 @@
--
-- The contents of this file are subject to the license and copyright
-- detailed in the LICENSE and NOTICE files at the root of the source
-- tree and available online at
--
-- http://www.dspace.org/license/
--
-- ===============================================================
-- WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
--
-- DO NOT MANUALLY RUN THIS DATABASE MIGRATION. IT WILL BE EXECUTED
-- AUTOMATICALLY (IF NEEDED) BY "FLYWAY" WHEN YOU STARTUP DSPACE.
-- http://flywaydb.org/
-- ===============================================================
CREATE SEQUENCE process_id_seq;
CREATE TABLE process
(
process_id INTEGER NOT NULL PRIMARY KEY,
user_id UUID NOT NULL,
start_time TIMESTAMP,
finished_time TIMESTAMP,
creation_time TIMESTAMP NOT NULL,
script VARCHAR(256) NOT NULL,
status VARCHAR(32),
parameters VARCHAR (512)
);
CREATE TABLE process2bitstream
(
process_id INTEGER REFERENCES process(process_id),
bitstream_id UUID REFERENCES bitstream(uuid),
CONSTRAINT PK_process2bitstream PRIMARY KEY (process_id, bitstream_id)
);
CREATE INDEX process_user_id_idx ON process(user_id);
CREATE INDEX process_status_idx ON process(status);
CREATE INDEX process_name_idx on process(script);
CREATE INDEX process_start_time_idx on process(start_time);

40
dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/oracle/V7.0_2019_06_14__scripts-and-process.sql Normal file
View File

@@ -0,0 +1,40 @@
--
-- The contents of this file are subject to the license and copyright
-- detailed in the LICENSE and NOTICE files at the root of the source
-- tree and available online at
--
-- http://www.dspace.org/license/
--
-- ===============================================================
-- WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
--
-- DO NOT MANUALLY RUN THIS DATABASE MIGRATION. IT WILL BE EXECUTED
-- AUTOMATICALLY (IF NEEDED) BY "FLYWAY" WHEN YOU STARTUP DSPACE.
-- http://flywaydb.org/
-- ===============================================================
CREATE SEQUENCE process_id_seq;
CREATE TABLE process
(
process_id INTEGER NOT NULL PRIMARY KEY,
user_id RAW(16) NOT NULL,
start_time TIMESTAMP,
finished_time TIMESTAMP,
creation_time TIMESTAMP NOT NULL,
script VARCHAR(256) NOT NULL,
status VARCHAR(32),
parameters VARCHAR (512)
);
CREATE TABLE process2bitstream
(
process_id INTEGER REFERENCES process(process_id),
bitstream_id RAW(16) REFERENCES bitstream(uuid),
CONSTRAINT PK_process2bitstream PRIMARY KEY (process_id, bitstream_id)
);
CREATE INDEX process_user_id_idx ON process(user_id);
CREATE INDEX process_status_idx ON process(status);
CREATE INDEX process_name_idx on process(script);
CREATE INDEX process_start_time_idx on process(start_time);

37
dspace-server-webapp/src/main/java/org/dspace/app/rest/ProcessRestController.java
View File

@@ -7,10 +7,14 @@
*/
package org.dspace.app.rest;
import static org.springframework.hateoas.mvc.ControllerLinkBuilder.linkTo;
import static org.springframework.hateoas.mvc.ControllerLinkBuilder.methodOn;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -24,7 +28,10 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.Pageable;
import org.springframework.data.web.PagedResourcesAssembler;
import org.springframework.hateoas.Link;
import org.springframework.hateoas.PagedResources;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
@@ -63,29 +70,31 @@ public class ProcessRestController implements InitializingBean {
* ProcessResource objects
* @throws Exception If something goes wrong
*/
@PreAuthorize("hasAuthority('ADMIN')")
@RequestMapping(method = RequestMethod.GET)
public EmbeddedPage getProcesses(Pageable pageable) throws Exception {
public PagedResources<ProcessResource> getProcesses(Pageable pageable, PagedResourcesAssembler assembler)
throws Exception {
if (log.isTraceEnabled()) {
log.trace("Retrieving processes");
}
List<ProcessRest> processRestList = processRestRepository.getAllProcesses();
List<ProcessResource> processResources = new LinkedList<>();
for (ProcessRest processRest : processRestList) {
ProcessResource processResource = new ProcessResource(processRest);
List<ProcessResource> processResources = processRestRepository.getAllProcesses(pageable).stream()
.map(processRest -> {
ProcessResource processResource =
new ProcessResource(
processRest);
linkService.addLinks(processResource);
processResources.add(processResource);
return processResource;
}).collect(Collectors.toList());
Page page = new PageImpl<>(processResources, pageable, processRestRepository.getTotalAmountOfProcesses());
Link link = linkTo(methodOn(this.getClass()).getProcesses(pageable, assembler)).withSelfRel();
PagedResources<ProcessResource> result = assembler.toResource(page, link);
return result;
}
Page page = new PageImpl<>(processResources, pageable, processRestList.size());
// SearchResultsResourceHalLinkFactory linkFactory = new SearchResultsResourceHalLinkFactory();
EmbeddedPage embeddedPage = new EmbeddedPage("test",
page, processResources, "scripts");
return embeddedPage;
}
/**
* This method will retrieve the Process object from the database that matches the given processId and it'll

3
dspace-server-webapp/src/main/java/org/dspace/app/rest/ScriptRestController.java
View File

@@ -29,9 +29,6 @@ public class ScriptRestController {
private static final Logger log = LogManager.getLogger();
@Autowired
private DiscoverableEndpointsService discoverableEndpointsService;
@Autowired
private ScriptRestRepository scriptRestRepository;

2
dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/processes/ParameterConverter.java
View File

@@ -33,7 +33,7 @@ public class ParameterConverter {
for (Option option : CollectionUtils.emptyIfNull(options.getOptions())) {
ParameterRest parameterRest = new ParameterRest();
parameterRest.setDescription(option.getDescription());
parameterRest.setName(option.getOpt());
parameterRest.setName("-" + (option.getOpt() != null ? option.getOpt() : "-" + option.getLongOpt()));
parameterRest.setType(((Class) option.getType()).getSimpleName());
listToReturn.add(parameterRest);
}

1
dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/processes/ProcessConverter.java
View File

@@ -37,6 +37,7 @@ public class ProcessConverter implements DSpaceConverter<Process, ProcessRest> {
processRest.setUserId(process.getEPerson().getID());
processRest.setProcessStatus(process.getProcessStatus());
processRest.setStartTime(process.getStartTime());
processRest.setEndTime(process.getFinishedTime());
processRest.setParameterRestList(
processService.getParameters(process).stream().map(x -> dSpaceRunnableParameterConverter.fromModel(x))
.collect(Collectors.toList()));

54
dspace-server-webapp/src/main/java/org/dspace/app/rest/link/process/ProcessHalLinkFactory.java
View File

@@ -1,53 +1,19 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.rest.link.process;
import java.util.LinkedList;
import org.dspace.app.rest.ProcessRestController;
import org.dspace.app.rest.link.HalLinkFactory;
import org.dspace.app.rest.model.hateoas.ProcessResource;
import org.dspace.services.ConfigurationService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Pageable;
import org.springframework.hateoas.Link;
import org.springframework.stereotype.Component;
import org.springframework.web.util.UriComponentsBuilder;
/**
* This HalLinkFactory class will provide a means to add Links to the ProcessResource
*/
@Component
public class ProcessHalLinkFactory extends HalLinkFactory<ProcessResource, ProcessRestController> {
public abstract class ProcessHalLinkFactory<T> extends HalLinkFactory<T, ProcessRestController> {
@Autowired
private ConfigurationService configurationService;
public UriComponentsBuilder buildProcessesBaseLink() {
try {
UriComponentsBuilder uriBuilder = uriBuilder(getMethodOn().getProcesses(null, null));
@Override
protected void addLinks(ProcessResource halResource, Pageable pageable, LinkedList<Link> list) throws Exception {
String dspaceRestUrl = configurationService.getProperty("dspace.restUrl");
list.add(buildLink(Link.REL_SELF, getMethodOn().getProcessById(halResource.getContent().getProcessId())));
list.add(
buildLink("script", dspaceRestUrl + "/api/system/scripts/" + halResource.getContent().getScriptName()));
// TODO Replace the bottom two with getMethodOn() when the Controller methods are created
list.add(buildLink("output", dspaceRestUrl + "/api/system/processes/" + halResource.getContent()
.getProcessId() + "/output"
));
list.add(buildLink("files", dspaceRestUrl + "/api/system/processes/" + halResource.getContent()
.getProcessId() + "/files"));
}
@Override
protected Class<ProcessRestController> getControllerClass() {
return ProcessRestController.class;
}
@Override
protected Class<ProcessResource> getResourceClass() {
return ProcessResource.class;
return uriBuilder;
} catch (Exception ex) {
//The method throwing the exception is never really executed, so this exception can never occur
return null;
}
}
}

40
dspace-server-webapp/src/main/java/org/dspace/app/rest/link/process/ProcessResourceHalLinkFactory.java Normal file
View File

@@ -0,0 +1,40 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.rest.link.process;
import java.util.LinkedList;
import org.dspace.app.rest.ProcessRestController;
import org.dspace.app.rest.model.hateoas.ProcessResource;
import org.dspace.services.ConfigurationService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Pageable;
import org.springframework.hateoas.Link;
import org.springframework.stereotype.Component;
@Component
public class ProcessResourceHalLinkFactory extends ProcessHalLinkFactory<ProcessResource> {
@Autowired
private ConfigurationService configurationService;
protected void addLinks(ProcessResource halResource, Pageable pageable, LinkedList<Link> list) throws Exception {
String dspaceRestUrl = configurationService.getProperty("dspace.restUrl");
list.add(buildLink(Link.REL_SELF, getMethodOn().getProcessById(halResource.getContent().getProcessId())));
list.add(buildLink("script", dspaceRestUrl + "/api/system/scripts/" + halResource.getContent().getScriptName()));
}
protected Class<ProcessRestController> getControllerClass() {
return ProcessRestController.class;
}
protected Class<ProcessResource> getResourceClass() {
return ProcessResource.class;
}
}

11
dspace-server-webapp/src/main/java/org/dspace/app/rest/model/ProcessRest.java
View File

@@ -12,6 +12,7 @@ import java.util.List;
import java.util.UUID;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.dspace.app.rest.ProcessRestController;
import org.dspace.content.ProcessStatus;
@@ -40,7 +41,9 @@ public class ProcessRest extends BaseObjectRest<UUID> {
private UUID userId;
private Integer processId;
private Date startTime;
private Date endTime;
private ProcessStatus processStatus;
@JsonProperty(value = "parameters")
private List<ParameterValueRest> parameterRestList;
public UUID getUserId() {
@@ -91,6 +94,14 @@ public class ProcessRest extends BaseObjectRest<UUID> {
this.scriptName = scriptName;
}
public Date getEndTime() {
return endTime;
}
public void setEndTime(Date endTime) {
this.endTime = endTime;
}
@JsonIgnore
@Override
public UUID getId() {

9
dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ProcessRestRepository.java
View File

@@ -17,6 +17,7 @@ import org.dspace.content.Process;
import org.dspace.content.service.ProcessService;
import org.dspace.core.Context;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Component;
/**
@@ -36,9 +37,9 @@ public class ProcessRestRepository extends AbstractDSpaceRestRepository {
* @return The list of ProcessRest objects coming forth from all Process objects in the database
* @throws SQLException If something goes wrong
*/
public List<ProcessRest> getAllProcesses() throws SQLException {
public List<ProcessRest> getAllProcesses(Pageable pageable) throws SQLException {
Context context = obtainContext();
List<Process> list = processService.findAll(context);
List<Process> list = processService.findAll(context, pageable.getPageSize(), pageable.getOffset());
List<ProcessRest> listToReturn = new LinkedList<>();
for (Process process : list) {
@@ -60,4 +61,8 @@ public class ProcessRestRepository extends AbstractDSpaceRestRepository {
Process process = processService.find(context, processId);
return processConverter.fromModel(process);
}
public int getTotalAmountOfProcesses() throws SQLException {
return processService.countTotal(obtainContext());
}
}

104
dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ScriptRestRepository.java
View File

@@ -30,6 +30,7 @@ import org.dspace.app.rest.model.ScriptRest;
import org.dspace.app.rest.model.hateoas.DSpaceResource;
import org.dspace.app.rest.model.hateoas.ScriptResource;
import org.dspace.app.rest.scripts.handler.impl.RestDSpaceRunnableHandler;
import org.dspace.authorize.AuthorizeException;
import org.dspace.content.service.ProcessService;
import org.dspace.core.Context;
import org.dspace.scripts.DSpaceCommandLineParameter;
@@ -40,6 +41,7 @@ import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Component;
import org.springframework.web.multipart.MultipartFile;
/**
* This is the REST repository dealing with the Script logic
@@ -100,37 +102,23 @@ public class ScriptRestRepository extends DSpaceRestRepository<ScriptRest, Strin
* @throws SQLException If something goes wrong
* @throws IOException If something goes wrong
*/
public ProcessRest startProcess(String scriptName) throws SQLException, IOException {
public ProcessRest startProcess(String scriptName) throws SQLException, IOException, AuthorizeException {
Context context = obtainContext();
List<ParameterValueRest> parameterValueRestList = new LinkedList<>();
ObjectMapper objectMapper = new ObjectMapper();
String propertiesJson = requestService.getCurrentRequest().getServletRequest().getParameter("properties");
if (StringUtils.isNotBlank(propertiesJson)) {
try {
parameterValueRestList = Arrays
.asList(objectMapper.readValue(propertiesJson, ParameterValueRest[].class));
} catch (IOException e) {
log.error(
"Couldn't convert the given properties to proper ParameterValueRest objects: " + propertiesJson, e);
throw e;
String properties = requestService.getCurrentRequest().getServletRequest().getParameter("properties");
List<DSpaceCommandLineParameter> dSpaceCommandLineParameters =
processPropertiesToDSpaceCommandLineParameters(properties);
DSpaceRunnable scriptToExecute = getdSpaceRunnableForName(scriptName);
if (scriptToExecute == null) {
throw new DSpaceBadRequestException("The script for name: " + scriptName + " wasn't found");
}
if (!scriptToExecute.isAllowedToExecute(context)) {
throw new AuthorizeException("Current user is not eligible to execute script with name: " + scriptName);
}
List<DSpaceCommandLineParameter> dSpaceCommandLineParameters = new LinkedList<>();
dSpaceCommandLineParameters.addAll(
parameterValueRestList.stream().map(x -> dSpaceRunnableParameterConverter.toModel(x))
.collect(Collectors.toList()));
try {
RestDSpaceRunnableHandler restDSpaceRunnableHandler = new RestDSpaceRunnableHandler(
context.getCurrentUser(), scriptName, dSpaceCommandLineParameters);
List<String> args = new ArrayList<>();
args.add(scriptName);
for (DSpaceCommandLineParameter parameter : dSpaceCommandLineParameters) {
args.add(parameter.getName());
args.add(parameter.getValue());
}
runDSpaceScriptWithArgs(args.toArray(new String[0]), restDSpaceRunnableHandler, scriptName);
List<String> args = constructArgs(dSpaceCommandLineParameters);
try {
runDSpaceScript(scriptToExecute, restDSpaceRunnableHandler, args);
context.complete();
return processConverter.fromModel(restDSpaceRunnableHandler.getProcess());
} catch (SQLException e) {
@@ -140,29 +128,55 @@ public class ScriptRestRepository extends DSpaceRestRepository<ScriptRest, Strin
}
return null;
}
private List<DSpaceCommandLineParameter> processPropertiesToDSpaceCommandLineParameters(String propertiesJson)
throws IOException {
List<ParameterValueRest> parameterValueRestList = new LinkedList<>();
ObjectMapper objectMapper = new ObjectMapper();
if (StringUtils.isNotBlank(propertiesJson)) {
parameterValueRestList = Arrays.asList(objectMapper.readValue(propertiesJson, ParameterValueRest[].class));
}
/**
* This method will try to find the script to run and run if possible
* @param args The arguments to be passed along to the script
* @param dSpaceRunnableHandler The handler for the script, this will be a RestDSpaceRunnableHandler in this
* instance
* @param scriptName The name of the script for which a process wants to be started
*/
private void runDSpaceScriptWithArgs(String[] args, RestDSpaceRunnableHandler dSpaceRunnableHandler,
String scriptName) {
List<DSpaceRunnable> scripts = new DSpace().getServiceManager().getServicesByType(DSpaceRunnable.class);
for (DSpaceRunnable script : scripts) {
List<DSpaceCommandLineParameter> dSpaceCommandLineParameters = new LinkedList<>();
dSpaceCommandLineParameters.addAll(
parameterValueRestList.stream().map(x -> dSpaceRunnableParameterConverter.toModel(x))
.collect(Collectors.toList()));
return dSpaceCommandLineParameters;
}
private DSpaceRunnable getdSpaceRunnableForName(String scriptName) {
DSpaceRunnable scriptToExecute = null;
for (DSpaceRunnable script : dspaceRunnables) {
if (StringUtils.equalsIgnoreCase(script.getName(), scriptName)) {
scriptToExecute = script;
break;
}
}
return scriptToExecute;
}
private List<String> constructArgs(List<DSpaceCommandLineParameter> dSpaceCommandLineParameters) {
List<String> args = new ArrayList<>();
for (DSpaceCommandLineParameter parameter : dSpaceCommandLineParameters) {
args.add(parameter.getName());
if (parameter.getValue() != null) {
args.add(parameter.getValue());
}
}
return args;
}
private void runDSpaceScript(DSpaceRunnable scriptToExecute,
RestDSpaceRunnableHandler restDSpaceRunnableHandler, List<String> args) {
try {
script.initialize(args, dSpaceRunnableHandler);
dSpaceRunnableHandler.schedule(script);
scriptToExecute.initialize(args.toArray(new String[0]), restDSpaceRunnableHandler);
restDSpaceRunnableHandler.schedule(scriptToExecute);
} catch (ParseException e) {
script.printHelp();
dSpaceRunnableHandler
.handleException("Failed to parse the arguments given to the script with name: " + scriptName
+ " and args: " + Arrays.toString(args), e);
}
}
scriptToExecute.printHelp();
restDSpaceRunnableHandler
.handleException(
"Failed to parse the arguments given to the script with name: " + scriptToExecute.getName()
+ " and args: " + args, e);
}
}
}

7
dspace/config/launcher.xml
View File

@@ -138,13 +138,6 @@
<class>org.dspace.authority.indexer.AuthorityIndexClient</class>
</step>
</command>
<command>
<name>index-discovery</name>
<description>Update Discovery Solr Search Index</description>
<step>
<class>org.dspace.discovery.IndexClient</class>
</step>
</command>
<command>
<name>itemupdate</name>
<description>Item update tool for altering metadata and bitstream content in items</description>

2
dspace/config/spring/api/scripts-and-processes.xml
View File

@@ -4,7 +4,7 @@
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-2.5.xsd">
<bean id="indexClient" class="org.dspace.scripts.impl.IndexClient" scope="prototype">
<bean id="indexClient" class="org.dspace.discovery.IndexClient" scope="prototype">
<property name="name" value="index-discovery"/>
<property name="description" value="Update Discovery Solr Search Index"/>
</bean>