hazza/DSpace
1
0
Fork 0
mirror of https://github.com/DSpace/DSpace.git synced 2025-10-07 01:54:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

intermittent commit

Browse Source
This commit is contained in:
Yana De Pauw
2020-08-14 10:15:06 +02:00
parent d1ee942bd4
commit 26935f9d23
12 changed files with 286 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
dspace-api/src/main/java/org/dspace/scripts/Process.java
View File

@@ -82,6 +82,7 @@ public class Process implements ReloadableEntity<Integer> {
private Date creationTime;
public static final String BITSTREAM_TYPE_METADATAFIELD = "dspace.process.filetype";
public static final String OUTPUT_TYPE = "script_output";
protected Process() {
}

14
dspace-api/src/main/java/org/dspace/scripts/ProcessLogLevel.java Normal file
View File

@@ -0,0 +1,14 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.scripts;
public enum ProcessLogLevel {
INFO,
WARNING,
ERROR
}

53
dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java
View File

@@ -7,9 +7,16 @@
*/
package org.dspace.scripts;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.Writer;
import java.nio.Buffer;
import java.sql.SQLException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@@ -20,6 +27,7 @@ import java.util.Set;
import java.util.regex.Pattern;
import org.apache.commons.collections4.ListUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.Logger;
import org.dspace.authorize.AuthorizeException;
@@ -37,6 +45,7 @@ import org.dspace.core.Constants;
import org.dspace.core.Context;
import org.dspace.core.LogManager;
import org.dspace.eperson.EPerson;
import org.dspace.eperson.service.EPersonService;
import org.dspace.scripts.service.ProcessService;
import org.springframework.beans.factory.annotation.Autowired;
@@ -62,6 +71,9 @@ public class ProcessServiceImpl implements ProcessService {
@Autowired
private MetadataFieldService metadataFieldService;
@Autowired
private EPersonService ePersonService;
@Override
public Process create(Context context, EPerson ePerson, String scriptName,
List<DSpaceCommandLineParameter> parameters) throws SQLException {
@@ -245,4 +257,45 @@ public class ProcessServiceImpl implements ProcessService {
return new ArrayList<>(fileTypesSet);
}
@Override
public void appendLog(int processId, String scriptName, String output, ProcessLogLevel processLogLevel)
throws IOException {
File tmpDir = FileUtils.getTempDirectory();
File tempFile = new File(tmpDir, scriptName + processId + ".log");
FileWriter out = new FileWriter(tempFile, true);
try {
try (BufferedWriter writer = new BufferedWriter(out)) {
writer.append(formatLogLine(processId, scriptName, output, processLogLevel));
writer.newLine();
}
} finally {
out.close();
}
}
public void createLogBitstream(Context context, Process process)
throws IOException, SQLException, AuthorizeException {
File tmpDir = FileUtils.getTempDirectory();
File tempFile = new File(tmpDir, process.getName() + process.getID() + ".log");
FileInputStream inputStream = FileUtils.openInputStream(tempFile);
appendFile(context, process, inputStream, Process.OUTPUT_TYPE, process.getName() + process.getID() + ".log");
inputStream.close();
tempFile.delete();
}
private String formatLogLine(int processId, String scriptName, String output, ProcessLogLevel processLogLevel) {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
StringBuilder sb = new StringBuilder();
sb.append(sdf.format(new Date()));
sb.append(" ");
sb.append(processLogLevel);
sb.append(" ");
sb.append(scriptName);
sb.append(" - ");
sb.append(processId);
sb.append(" @ ");
sb.append(output);
return sb.toString();
}
}

4
dspace-api/src/main/java/org/dspace/scripts/service/ProcessService.java
View File

@@ -18,6 +18,7 @@ import org.dspace.core.Context;
import org.dspace.eperson.EPerson;
import org.dspace.scripts.DSpaceCommandLineParameter;
import org.dspace.scripts.Process;
import org.dspace.scripts.ProcessLogLevel;
/**
* An interface for the ProcessService with methods regarding the Process workload
@@ -189,4 +190,7 @@ public interface ProcessService {
*/
public List<String> getFileTypesForProcessBitstreams(Context context, Process process);
void appendLog(int processId, String scriptName, String output, ProcessLogLevel processLogLevel) throws IOException;
void createLogBitstream(Context context, Process process)
throws IOException, SQLException, AuthorizeException;
}

3
dspace-api/src/test/data/dspaceFolder/config/spring/api/scripts.xml
View File

@@ -4,6 +4,9 @@
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-2.5.xsd">
<bean id="dspaceRunnableThreadExecutor" class="org.springframework.core.task.SyncTaskExecutor"/>
<bean id="index-discovery" class="org.dspace.discovery.IndexDiscoveryScriptConfiguration">
<property name="description" value="Update Discovery Solr Search Index"/>
<property name="dspaceRunnableClass" value="org.dspace.discovery.IndexClient"/>

2
dspace-api/src/test/java/org/dspace/eperson/GroupTest.java
View File

@@ -171,7 +171,7 @@ public class GroupTest extends AbstractUnitTest {
public void findAll() throws SQLException {
List<Group> groups = groupService.findAll(context, null);
assertThat("findAll 1", groups, notNullValue());
System.out.println("TEST GROUP OUTPUT " + groups);
System.out.println("TEST GROUP OUTPUT_TYPE " + groups);
assertTrue("findAll 2", 0 < groups.size());
}

6
dspace-server-webapp/src/main/java/org/dspace/app/rest/model/ProcessRest.java
View File

@@ -28,6 +28,10 @@ import org.dspace.scripts.Process;
@LinkRest(
name = ProcessRest.FILE_TYPES,
method = "getFileTypesFromProcess"
),
@LinkRest(
name = ProcessRest.OUTPUT,
method = "getOutputFromProcess"
)
})
public class ProcessRest extends BaseObjectRest<Integer> {
@@ -37,6 +41,8 @@ public class ProcessRest extends BaseObjectRest<Integer> {
public static final String FILES = "files";
public static final String FILE_TYPES = "filetypes";
public static final String OUTPUT = "output";
public String getCategory() {
return CATEGORY;
}

65
dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ProcessOutputLinkRepository.java Normal file
View File

@@ -0,0 +1,65 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.rest.repository;
import java.sql.SQLException;
import javax.annotation.Nullable;
import javax.servlet.http.HttpServletRequest;
import org.dspace.app.rest.model.BitstreamRest;
import org.dspace.app.rest.model.ProcessRest;
import org.dspace.app.rest.projection.Projection;
import org.dspace.authorize.AuthorizeException;
import org.dspace.authorize.service.AuthorizeService;
import org.dspace.content.Bitstream;
import org.dspace.core.Context;
import org.dspace.scripts.Process;
import org.dspace.scripts.service.ProcessService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Pageable;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Component;
@Component(ProcessRest.CATEGORY + "." + ProcessRest.NAME + "." + ProcessRest.OUTPUT)
public class ProcessOutputLinkRepository extends AbstractDSpaceRestRepository implements LinkRestRepository {
@Autowired
private ProcessService processService;
@Autowired
private AuthorizeService authorizeService;
// /**
// * This method will retrieve the list of {@link ProcessLog} objects from the {@link Process} as defined through
// the
// * given ID in the rest call and it'll wrap this in a {@link ProcessOutputRest} object to return these
// * @param request The current request
// * @param processId The given processId for the {@link Process}
// * @param optionalPageable Pageable if applicable
// * @param projection The current projection
// * @return The {@link ProcessOutputRest} containing the list of all {@link ProcessLog} for the
// * given {@link Process}
// * @throws SQLException If something goes wrong
// * @throws AuthorizeException If something goes wrong
// */
@PreAuthorize("hasAuthority('ADMIN')")
public BitstreamRest getOutputFromProcess(@Nullable HttpServletRequest request,
Integer processId,
@Nullable Pageable optionalPageable,
Projection projection) throws SQLException, AuthorizeException {
Context context = obtainContext();
Process process = processService.find(context, processId);
if ((context.getCurrentUser() == null) || (!context.getCurrentUser().equals(process.getEPerson())
&& !authorizeService.isAdmin(context))) {
throw new AuthorizeException("The current user is not eligible to view the process with id: " + processId);
}
Bitstream bitstream = processService.getBitstream(context, process, Process.OUTPUT_TYPE);
return converter.toRest(bitstream, projection);
}
}

46
dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java
View File

@@ -14,6 +14,7 @@ import java.io.StringWriter;
import java.sql.SQLException;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
@@ -26,9 +27,12 @@ import org.dspace.content.factory.ContentServiceFactory;
import org.dspace.content.service.BitstreamService;
import org.dspace.core.Context;
import org.dspace.eperson.EPerson;
import org.dspace.eperson.factory.EPersonServiceFactory;
import org.dspace.eperson.service.EPersonService;
import org.dspace.scripts.DSpaceCommandLineParameter;
import org.dspace.scripts.DSpaceRunnable;
import org.dspace.scripts.Process;
import org.dspace.scripts.ProcessLogLevel;
import org.dspace.scripts.factory.ScriptServiceFactory;
import org.dspace.scripts.handler.DSpaceRunnableHandler;
import org.dspace.scripts.service.ProcessService;
@@ -44,9 +48,11 @@ public class RestDSpaceRunnableHandler implements DSpaceRunnableHandler {
private BitstreamService bitstreamService = ContentServiceFactory.getInstance().getBitstreamService();
private ProcessService processService = ScriptServiceFactory.getInstance().getProcessService();
private EPersonService ePersonService = EPersonServiceFactory.getInstance().getEPersonService();
private Integer processId;
private String scriptName;
private UUID ePersonId;
/**
* This constructor will initialise the handler with the process created from the parameters
@@ -57,6 +63,7 @@ public class RestDSpaceRunnableHandler implements DSpaceRunnableHandler {
public RestDSpaceRunnableHandler(EPerson ePerson, String scriptName, List<DSpaceCommandLineParameter> parameters) {
Context context = new Context();
try {
ePersonId = ePerson.getID();
Process process = processService.create(context, ePerson, scriptName, parameters);
processId = process.getID();
this.scriptName = process.getName();
@@ -96,11 +103,20 @@ public class RestDSpaceRunnableHandler implements DSpaceRunnableHandler {
try {
Process process = processService.find(context, processId);
processService.complete(context, process);
context.complete();
logInfo("The script has completed");
EPerson ePerson = ePersonService.find(context, ePersonId);
context.setCurrentUser(ePerson);
processService.createLogBitstream(context, process);
context.complete();
} catch (SQLException e) {
log.error("RestDSpaceRunnableHandler with process: " + processId + " could not be completed", e);
} finally {
} catch (IOException | AuthorizeException e) {
log.error("RestDSpaceRunnableHandler with process: " + processId + " could not be completed due to an " +
"error with the logging bitstream", e);
} catch (Exception e) {
log.error(e.getMessage(), e);
}finally {
if (context.isValid()) {
context.abort();
}
@@ -130,9 +146,19 @@ public class RestDSpaceRunnableHandler implements DSpaceRunnableHandler {
try {
Process process = processService.find(context, processId);
processService.fail(context, process);
EPerson ePerson = ePersonService.find(context, ePersonId);
context.setCurrentUser(ePerson);
processService.createLogBitstream(context, process);
context.complete();
} catch (SQLException sqlException) {
log.error("SQL exception while handling another exception", e);
} catch (IOException | AuthorizeException ioException) {
log.error("RestDSpaceRunnableHandler with process: " + processId + " could not be completed due to an " +
"error with the logging bitstream", e);
} catch (Exception whatev) {
log.error(e.getMessage(), whatev);
} finally {
if (context.isValid()) {
context.abort();
@@ -156,18 +182,26 @@ public class RestDSpaceRunnableHandler implements DSpaceRunnableHandler {
String logMessage = getLogMessage(message);
log.info(logMessage);
appendLogToProcess(message, ProcessLogLevel.INFO);
}
@Override
public void logWarning(String message) {
String logMessage = getLogMessage(message);
log.warn(logMessage);
appendLogToProcess(message, ProcessLogLevel.INFO);
}
@Override
public void logError(String message) {
String logMessage = getLogMessage(message);
log.error(logMessage);
appendLogToProcess(message, ProcessLogLevel.ERROR);
}
@Override
@@ -249,4 +283,12 @@ public class RestDSpaceRunnableHandler implements DSpaceRunnableHandler {
}
taskExecutor.execute(script);
}
private void appendLogToProcess(String message, ProcessLogLevel error) {
try {
processService.appendLog(processId, scriptName, message, error);
} catch (IOException e) {
log.error("RestDSpaceRunnableHandler with process: " + processId + " could not write log to process", e);
}
}
}

45
dspace-server-webapp/src/test/java/org/dspace/app/rest/ProcessRestRepositoryIT.java
View File

@@ -31,6 +31,7 @@ import org.dspace.content.Bitstream;
import org.dspace.content.ProcessStatus;
import org.dspace.scripts.DSpaceCommandLineParameter;
import org.dspace.scripts.Process;
import org.dspace.scripts.ProcessLogLevel;
import org.dspace.scripts.service.ProcessService;
import org.hamcrest.Matchers;
import org.junit.After;
@@ -322,6 +323,50 @@ public class ProcessRestRepositoryIT extends AbstractControllerIntegrationTest {
}
// @Test
// public void getProcessOutput() throws Exception {
// try (InputStream is = IOUtils.toInputStream("Test File For Process", CharEncoding.UTF_8)) {
// processService.appendLog(process.getID(), process.getName(), "testlog", ProcessLogLevel.INFO);
// }
//
// List<String> fileTypesToCheck = new LinkedList<>();
// fileTypesToCheck.add("inputfile");
//
// String token = getAuthToken(admin.getEmail(), password);
//
// getClient(token).perform(get("/api/system/processes/" + process.getID() + "/output"))
// .andExpect(status().isOk())
// .andExpect(jsonPath("$.logs", containsInAnyOrder("testlog")))
// .andExpect(jsonPath("$.type", is("processOutput")));
//
//
// }
// @Test
// public void getProcessOutput() throws Exception {
// String token = getAuthToken(admin.getEmail(), password);
//
// Process process = ProcessBuilder.createProcess(context, admin, "mock-script", new LinkedList<>()).build();
//
// getClient(token).perform(get("/api/system/processes/" + process.getID()))
// .andExpect(status().isOk())
// .andExpect(jsonPath("$", Matchers.is(
// ProcessMatcher.matchProcess(process.getName(), String.valueOf(process.getEPerson().getID()),
// process.getID(), new LinkedList<>(), ProcessStatus.SCHEDULED)))
// );
//
// List<String> fileTypesToCheck = new LinkedList<>();
// fileTypesToCheck.add("inputfile");
//
// String token = getAuthToken(admin.getEmail(), password);
//
// getClient(token).perform(get("/api/system/processes/" + process.getID() + "/output"))
// .andExpect(status().isOk())
// .andExpect(jsonPath("$.logs", containsInAnyOrder("testlog")))
// .andExpect(jsonPath("$.type", is("processOutput")));
//
//
// }
@After
@Override
public void destroy() throws Exception {

46
dspace-server-webapp/src/test/java/org/dspace/app/rest/ScriptRestRepositoryIT.java
View File

@@ -288,6 +288,52 @@ public class ScriptRestRepositoryIT extends AbstractControllerIntegrationTest {
}
}
@Test
public void postProcessAndVerifyOutput() throws Exception {
LinkedList<DSpaceCommandLineParameter> parameters = new LinkedList<>();
parameters.add(new DSpaceCommandLineParameter("-r", "test"));
parameters.add(new DSpaceCommandLineParameter("-i", null));
List<ParameterValueRest> list = parameters.stream()
.map(dSpaceCommandLineParameter -> dSpaceRunnableParameterConverter
.convert(dSpaceCommandLineParameter, Projection.DEFAULT))
.collect(Collectors.toList());
String token = getAuthToken(admin.getEmail(), password);
List<ProcessStatus> acceptableProcessStatuses = new LinkedList<>();
acceptableProcessStatuses.addAll(Arrays.asList(ProcessStatus.SCHEDULED,
ProcessStatus.RUNNING,
ProcessStatus.COMPLETED));
AtomicReference<Integer> idRef = new AtomicReference<>();
try {
getClient(token)
.perform(post("/api/system/scripts/mock-script/processes").contentType("multipart/form-data")
.param("properties",
new Gson().toJson(list)))
.andExpect(status().isAccepted())
.andExpect(jsonPath("$", is(
ProcessMatcher.matchProcess("mock-script",
String.valueOf(admin.getID()),
parameters,
acceptableProcessStatuses))))
.andDo(result -> idRef
.set(read(result.getResponse().getContentAsString(), "$.processId")));
getClient(token).perform(get("/api/system/processes/" + idRef.get() + "/output"))
.andExpect(status().isOk())
.andExpect(jsonPath("$.logs", containsInAnyOrder("testlog")))
.andExpect(jsonPath("$.type", is("processOutput")));
} finally {
ProcessBuilder.deleteProcess(idRef.get());
}
}
@Test
public void postProcessAdminWithWrongContentTypeBadRequestException() throws Exception {

4
dspace-server-webapp/src/test/java/org/dspace/scripts/impl/MockDSpaceRunnableScript.java
View File

@@ -15,6 +15,10 @@ import org.dspace.utils.DSpace;
public class MockDSpaceRunnableScript extends DSpaceRunnable<MockDSpaceRunnableScriptConfiguration> {
@Override
public void internalRun() throws Exception {
handler.logInfo("Logging INFO for Mock DSpace Script");
handler.logError("Logging ERROR for Mock DSpace Script");
handler.logWarning("Logging WARNING for Mock DSpace Script");
handler.logDebug("Logging DEBUG for Mock DSpace Script");
}
@Override