Merge pull request #3133 from 4Science/DS-4443_improvements

[DS-4443] delete solr record on workspaceitem deletion (updated PR)
This commit is contained in:
Tim Donohue
2021-01-29 14:21:13 -06:00
committed by GitHub
6 changed files with 373 additions and 37 deletions

View File

@@ -130,6 +130,18 @@ public class IndexEventConsumer implements Consumer {
}
} else {
log.debug("consume() adding event to update queue: " + event.toString());
if (event.getSubjectType() == Constants.ITEM) {
// if it is an item we cannot know about its previous state, so it could be a
// workspaceitem that has been deposited right now or an approved/reject
// workflowitem.
// As the workflow is not necessary enabled it can happen than a workspaceitem
// became directly an item without giving us the chance to retrieve a
// workflowitem... so we need to force the unindex of all the related data
// before to index it again to be sure to don't leave any zombie in solr
String detail =
Constants.typeText[event.getSubjectType()] + "-" + event.getSubjectID().toString();
uniqueIdsToDelete.add(detail);
}
objectsToUpdate.addAll(indexObjectServiceFactory.getIndexableObjects(ctx, subject));
}
break;
@@ -151,7 +163,7 @@ public class IndexEventConsumer implements Consumer {
if (event.getSubjectType() == -1 || event.getSubjectID() == null) {
log.warn("got null subject type and/or ID on DELETE event, skipping it.");
} else {
String detail = event.getSubjectType() + "-" + event.getSubjectID().toString();
String detail = Constants.typeText[event.getSubjectType()] + "-" + event.getSubjectID().toString();
log.debug("consume() adding event to delete queue: " + event.toString());
uniqueIdsToDelete.add(detail);
}
@@ -175,6 +187,16 @@ public class IndexEventConsumer implements Consumer {
public void end(Context ctx) throws Exception {
try {
for (String uid : uniqueIdsToDelete) {
try {
indexer.unIndexContent(ctx, uid, false);
if (log.isDebugEnabled()) {
log.debug("UN-Indexed Item, handle=" + uid);
}
} catch (Exception e) {
log.error("Failed while UN-indexing object: " + uid, e);
}
}
// update the changed Items not deleted because they were on create list
for (IndexableObject iu : objectsToUpdate) {
/* we let all types through here and
@@ -183,7 +205,7 @@ public class IndexEventConsumer implements Consumer {
*/
iu.setIndexedObject(ctx.reloadEntity(iu.getIndexedObject()));
String uniqueIndexID = iu.getUniqueIndexID();
if (uniqueIndexID != null && !uniqueIdsToDelete.contains(uniqueIndexID)) {
if (uniqueIndexID != null) {
try {
indexer.indexContent(ctx, iu, true, false);
log.debug("Indexed "
@@ -195,17 +217,6 @@ public class IndexEventConsumer implements Consumer {
}
}
}
for (String uid : uniqueIdsToDelete) {
try {
indexer.unIndexContent(ctx, uid, false);
if (log.isDebugEnabled()) {
log.debug("UN-Indexed Item, handle=" + uid);
}
} catch (Exception e) {
log.error("Failed while UN-indexing object: " + uid, e);
}
}
} finally {
if (!objectsToUpdate.isEmpty() || !uniqueIdsToDelete.isEmpty()) {

View File

@@ -68,7 +68,7 @@ public abstract class IndexObjectFactoryFactory {
*/
public IndexFactory getIndexFactoryByType(String indexableFactoryType) {
for (IndexFactory indexableObjectFactory : getIndexFactories()) {
if (indexableObjectFactory.getType().equals(indexableFactoryType)) {
if (StringUtils.equalsIgnoreCase(indexableObjectFactory.getType(), indexableFactoryType)) {
return indexableObjectFactory;
}
}

View File

@@ -84,6 +84,10 @@ loglevel.dspace = INFO
###########################################
# CUSTOM UNIT / INTEGRATION TEST SETTINGS #
###########################################
# custom dispatcher to be used by dspace-api IT that doesn't need SOLR
event.dispatcher.exclude-discovery.class = org.dspace.event.BasicDispatcher
event.dispatcher.exclude-discovery.consumers = versioning, eperson
# Configure authority control for Unit Testing (in DSpaceControlledVocabularyTest)
# (This overrides default, commented out settings in dspace.cfg)
plugin.selfnamed.org.dspace.content.authority.ChoiceAuthority = \

View File

@@ -11,7 +11,6 @@ import static org.junit.Assert.fail;
import java.sql.SQLException;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.logging.log4j.Logger;
import org.dspace.authorize.AuthorizeException;
import org.dspace.authorize.factory.AuthorizeServiceFactory;
@@ -21,8 +20,6 @@ import org.dspace.core.I18nUtil;
import org.dspace.eperson.EPerson;
import org.dspace.eperson.factory.EPersonServiceFactory;
import org.dspace.eperson.service.EPersonService;
import org.dspace.services.ConfigurationService;
import org.dspace.services.factory.DSpaceServicesFactory;
import org.dspace.storage.rdbms.DatabaseUtils;
import org.junit.After;
import org.junit.Before;
@@ -123,8 +120,11 @@ public class AbstractUnitTest extends AbstractDSpaceTest {
context.restoreAuthSystemState();
// Ensure all tests run with Solr indexing disabled
disableSolrIndexing();
// we turn this off because
// Solr is NOT used in the OLD dspace-api test framework. Instead, Solr/Discovery indexing is
// exercised in the new Integration Tests (which use an embedded Solr) and extends
// org.dspace.AbstractIntegrationTestWithDatabase
context.setDispatcher("exclude-discovery");
} catch (AuthorizeException ex) {
log.error("Error creating initial eperson or default groups", ex);
fail("Error creating initial eperson or default groups in AbstractUnitTest init()");
@@ -167,22 +167,4 @@ public class AbstractUnitTest extends AbstractDSpaceTest {
}
}
/**
* Utility method which ensures Solr indexing is DISABLED in all Tests. We turn this off because
* Solr is NOT used in the dspace-api test framework. Instead, Solr/Discovery indexing is
* exercised in the dspace-server Integration Tests (which use an embedded Solr).
*/
protected static void disableSolrIndexing() {
// Get our currently configured list of event consumers
ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService();
String[] consumers = configurationService.getArrayProperty("event.dispatcher.default.consumers");
// Remove "discovery" from the configured consumers (if it exists).
// This turns off Discovery/Solr indexing after any object changes.
if (ArrayUtils.contains(consumers, "discovery")) {
consumers = ArrayUtils.removeElement(consumers, "discovery");
configurationService.setProperty("event.dispatcher.default.consumers", consumers);
}
}
}

View File

@@ -22,6 +22,7 @@ import org.dspace.content.WorkspaceItem;
import org.dspace.content.service.WorkspaceItemService;
import org.dspace.core.Context;
import org.dspace.eperson.EPerson;
import org.dspace.xmlworkflow.storedcomponents.XmlWorkflowItem;
/**
* Builder to construct WorkspaceItem objects
@@ -110,6 +111,13 @@ public class WorkspaceItemBuilder extends AbstractBuilder<WorkspaceItem, Workspa
workspaceItem = c.reloadEntity(workspaceItem);
if (workspaceItem != null) {
delete(c, workspaceItem);
} else {
item = c.reloadEntity(item);
// check if the wsi has been pushed to the workflow
XmlWorkflowItem wfi = workflowItemService.findByItem(c, item);
if (wfi != null) {
workflowItemService.delete(c, wfi);
}
}
item = c.reloadEntity(item);
if (item != null) {

View File

@@ -0,0 +1,331 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.discovery;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.sql.SQLException;
import java.util.List;
import javax.servlet.http.HttpServletRequest;
import org.dspace.AbstractIntegrationTestWithDatabase;
import org.dspace.authorize.AuthorizeException;
import org.dspace.builder.ClaimedTaskBuilder;
import org.dspace.builder.CollectionBuilder;
import org.dspace.builder.CommunityBuilder;
import org.dspace.builder.PoolTaskBuilder;
import org.dspace.builder.WorkflowItemBuilder;
import org.dspace.builder.WorkspaceItemBuilder;
import org.dspace.content.Collection;
import org.dspace.content.Community;
import org.dspace.content.WorkspaceItem;
import org.dspace.content.factory.ContentServiceFactory;
import org.dspace.content.service.ItemService;
import org.dspace.content.service.WorkspaceItemService;
import org.dspace.discovery.indexobject.IndexableClaimedTask;
import org.dspace.discovery.indexobject.IndexableItem;
import org.dspace.discovery.indexobject.IndexablePoolTask;
import org.dspace.discovery.indexobject.IndexableWorkflowItem;
import org.dspace.discovery.indexobject.IndexableWorkspaceItem;
import org.dspace.eperson.EPerson;
import org.dspace.services.factory.DSpaceServicesFactory;
import org.dspace.workflow.WorkflowException;
import org.dspace.xmlworkflow.WorkflowConfigurationException;
import org.dspace.xmlworkflow.factory.XmlWorkflowServiceFactory;
import org.dspace.xmlworkflow.service.WorkflowRequirementsService;
import org.dspace.xmlworkflow.service.XmlWorkflowService;
import org.dspace.xmlworkflow.state.Step;
import org.dspace.xmlworkflow.state.Workflow;
import org.dspace.xmlworkflow.state.actions.WorkflowActionConfig;
import org.dspace.xmlworkflow.storedcomponents.ClaimedTask;
import org.dspace.xmlworkflow.storedcomponents.PoolTask;
import org.dspace.xmlworkflow.storedcomponents.XmlWorkflowItem;
import org.dspace.xmlworkflow.storedcomponents.service.ClaimedTaskService;
import org.junit.Test;
import org.springframework.mock.web.MockHttpServletRequest;
/**
* This class will aim to test Discovery related use cases
*/
public class DiscoveryIT extends AbstractIntegrationTestWithDatabase {
protected WorkspaceItemService workspaceItemService = ContentServiceFactory.getInstance().getWorkspaceItemService();
protected SearchService searchService = SearchUtils.getSearchService();
XmlWorkflowService workflowService = XmlWorkflowServiceFactory.getInstance().getXmlWorkflowService();
WorkflowRequirementsService workflowRequirementsService = XmlWorkflowServiceFactory.getInstance().
getWorkflowRequirementsService();
ClaimedTaskService claimedTaskService = XmlWorkflowServiceFactory.getInstance().getClaimedTaskService();
ItemService itemService = ContentServiceFactory.getInstance().getItemService();
IndexingService indexer = DSpaceServicesFactory.getInstance().getServiceManager()
.getServiceByName(IndexingService.class.getName(),
IndexingService.class);
@Test
public void solrRecordsAfterDepositOrDeletionOfWorkspaceItemTest() throws Exception {
context.turnOffAuthorisationSystem();
Community community = CommunityBuilder.createCommunity(context)
.withName("Parent Community")
.build();
Collection col = CollectionBuilder.createCollection(context, community)
.withName("Collection without workflow")
.build();
Collection colWithWorkflow = CollectionBuilder.createCollection(context, community)
.withName("Collection WITH workflow")
.withWorkflowGroup(1, admin)
.build();
WorkspaceItem workspaceItem = WorkspaceItemBuilder.createWorkspaceItem(context, col)
.withTitle("No workflow")
.withAbstract("headache")
.build();
WorkspaceItem anotherWorkspaceItem = WorkspaceItemBuilder.createWorkspaceItem(context, col)
.withTitle("Another WS Item in No workflow collection")
.withAbstract("headache")
.build();
WorkspaceItem workspaceItemInWfCollection = WorkspaceItemBuilder.createWorkspaceItem(context, colWithWorkflow)
.withTitle("WS Item in workflow collection")
.withAbstract("headache")
.build();
context.restoreAuthSystemState();
// we start with 3 ws items
assertSearchQuery(IndexableWorkspaceItem.TYPE, 3);
// simulate the deposit
deposit(workspaceItem);
// now we should have 1 archived item and 2 ws items, no wf items or tasks
assertSearchQuery(IndexableWorkflowItem.TYPE, 0);
assertSearchQuery(IndexablePoolTask.TYPE, 0);
assertSearchQuery(IndexableClaimedTask.TYPE, 0);
assertSearchQuery(IndexableWorkspaceItem.TYPE, 2);
assertSearchQuery(IndexableItem.TYPE, 1);
// simulate the deposit of the ws item in the workflow collection
deposit(workspaceItemInWfCollection);
// now we should have 1 wf, 1 pool task, 1 ws item and 1 item
assertSearchQuery(IndexableWorkflowItem.TYPE, 1);
assertSearchQuery(IndexablePoolTask.TYPE, 1);
assertSearchQuery(IndexableClaimedTask.TYPE, 0);
assertSearchQuery(IndexableWorkspaceItem.TYPE, 1);
assertSearchQuery(IndexableItem.TYPE, 1);
// simulate the delete of last workspace item
deleteSubmission(anotherWorkspaceItem);
assertSearchQuery(IndexableWorkflowItem.TYPE, 1);
assertSearchQuery(IndexablePoolTask.TYPE, 1);
assertSearchQuery(IndexableClaimedTask.TYPE, 0);
assertSearchQuery(IndexableWorkspaceItem.TYPE, 0);
assertSearchQuery(IndexableItem.TYPE, 1);
}
@Test
public void solrRecordsAfterDealingWithWorkflowTest() throws Exception {
context.turnOffAuthorisationSystem();
Community community = CommunityBuilder.createCommunity(context)
.withName("Parent Community")
.build();
Collection collection = CollectionBuilder.createCollection(context, community)
.withWorkflowGroup(1, admin)
.build();
Workflow workflow = XmlWorkflowServiceFactory.getInstance().getWorkflowFactory().getWorkflow(collection);
ClaimedTask taskToApprove = ClaimedTaskBuilder.createClaimedTask(context, collection, admin)
.withTitle("Test workflow item to approve")
.withIssueDate("2019-03-06")
.withSubject("ExtraEntry")
.build();
ClaimedTask taskToReject = ClaimedTaskBuilder.createClaimedTask(context, collection, admin)
.withTitle("Test workflow item to reject")
.withIssueDate("2019-03-06")
.withSubject("ExtraEntry")
.build();
PoolTask taskToClaim = PoolTaskBuilder.createPoolTask(context, collection, admin)
.withTitle("Test pool task to claim")
.withIssueDate("2019-03-06")
.withSubject("ExtraEntry")
.build();
ClaimedTask taskToUnclaim = ClaimedTaskBuilder.createClaimedTask(context, collection, admin)
.withTitle("Test claimed task to unclaim")
.withIssueDate("2019-03-06")
.withSubject("ExtraEntry")
.build();
XmlWorkflowItem wfiToDelete = WorkflowItemBuilder.createWorkflowItem(context, collection)
.withTitle("Test workflow item to return")
.withIssueDate("2019-03-06")
.withSubject("ExtraEntry")
.build();
context.restoreAuthSystemState();
// we start with 5 workflow items, 3 claimed tasks, 2 pool task
assertSearchQuery(IndexableWorkflowItem.TYPE, 5);
assertSearchQuery(IndexableClaimedTask.TYPE, 3);
assertSearchQuery(IndexablePoolTask.TYPE, 2);
assertSearchQuery(IndexableWorkspaceItem.TYPE, 0);
assertSearchQuery(IndexableItem.TYPE, 0);
// claim
claim(workflow, taskToClaim, admin);
assertSearchQuery(IndexableWorkflowItem.TYPE, 5);
assertSearchQuery(IndexableClaimedTask.TYPE, 4);
assertSearchQuery(IndexablePoolTask.TYPE, 1);
assertSearchQuery(IndexableWorkspaceItem.TYPE, 0);
assertSearchQuery(IndexableItem.TYPE, 0);
// unclaim
returnClaimedTask(taskToUnclaim);
assertSearchQuery(IndexableWorkflowItem.TYPE, 5);
assertSearchQuery(IndexableClaimedTask.TYPE, 3);
assertSearchQuery(IndexablePoolTask.TYPE, 2);
assertSearchQuery(IndexableWorkspaceItem.TYPE, 0);
assertSearchQuery(IndexableItem.TYPE, 0);
// reject
MockHttpServletRequest httpRejectRequest = new MockHttpServletRequest();
httpRejectRequest.setParameter("submit_reject", "submit_reject");
httpRejectRequest.setParameter("reason", "test");
executeWorkflowAction(httpRejectRequest, workflow, taskToReject);
assertSearchQuery(IndexableWorkflowItem.TYPE, 4);
assertSearchQuery(IndexableClaimedTask.TYPE, 2);
assertSearchQuery(IndexablePoolTask.TYPE, 2);
assertSearchQuery(IndexableWorkspaceItem.TYPE, 1);
assertSearchQuery(IndexableItem.TYPE, 0);
// approve
MockHttpServletRequest httpApproveRequest = new MockHttpServletRequest();
httpApproveRequest.setParameter("submit_approve", "submit_approve");
executeWorkflowAction(httpApproveRequest, workflow, taskToApprove);
assertSearchQuery(IndexableWorkflowItem.TYPE, 3);
assertSearchQuery(IndexableClaimedTask.TYPE, 1);
assertSearchQuery(IndexablePoolTask.TYPE, 2);
assertSearchQuery(IndexableWorkspaceItem.TYPE, 1);
assertSearchQuery(IndexableItem.TYPE, 1);
// abort pool task
// as we have already unclaimed this task it is a pool task now
abort(taskToUnclaim.getWorkflowItem());
assertSearchQuery(IndexableWorkflowItem.TYPE, 2);
assertSearchQuery(IndexableClaimedTask.TYPE, 1);
assertSearchQuery(IndexablePoolTask.TYPE, 1);
assertSearchQuery(IndexableWorkspaceItem.TYPE, 2);
assertSearchQuery(IndexableItem.TYPE, 1);
// abort claimed task
// as we have already claimed this task it is a claimed task now
abort(taskToClaim.getWorkflowItem());
assertSearchQuery(IndexableWorkflowItem.TYPE, 1);
assertSearchQuery(IndexableClaimedTask.TYPE, 0);
assertSearchQuery(IndexablePoolTask.TYPE, 1);
assertSearchQuery(IndexableWorkspaceItem.TYPE, 3);
assertSearchQuery(IndexableItem.TYPE, 1);
// delete pool task / workflow item
deleteWorkflowItem(wfiToDelete);
assertSearchQuery(IndexableWorkflowItem.TYPE, 0);
assertSearchQuery(IndexableClaimedTask.TYPE, 0);
assertSearchQuery(IndexablePoolTask.TYPE, 0);
assertSearchQuery(IndexableWorkspaceItem.TYPE, 3);
assertSearchQuery(IndexableItem.TYPE, 1);
}
private void assertSearchQuery(String resourceType, int size) throws SearchServiceException {
DiscoverQuery discoverQuery = new DiscoverQuery();
discoverQuery.setQuery("*:*");
discoverQuery.addFilterQueries("search.resourcetype:" + resourceType);
DiscoverResult discoverResult = searchService.search(context, discoverQuery);
List<IndexableObject> indexableObjects = discoverResult.getIndexableObjects();
assertEquals(size, indexableObjects.size());
assertEquals(size, discoverResult.getTotalSearchResults());
}
private void deposit(WorkspaceItem workspaceItem)
throws SQLException, AuthorizeException, IOException, WorkflowException, SearchServiceException {
context.turnOffAuthorisationSystem();
workspaceItem = context.reloadEntity(workspaceItem);
XmlWorkflowItem workflowItem = workflowService.startWithoutNotify(context, workspaceItem);
context.commit();
indexer.commit();
context.restoreAuthSystemState();
}
private void deleteSubmission(WorkspaceItem anotherWorkspaceItem)
throws SQLException, AuthorizeException, IOException, SearchServiceException {
context.turnOffAuthorisationSystem();
anotherWorkspaceItem = context.reloadEntity(anotherWorkspaceItem);
workspaceItemService.deleteAll(context, anotherWorkspaceItem);
context.commit();
indexer.commit();
context.restoreAuthSystemState();
}
private void deleteWorkflowItem(XmlWorkflowItem workflowItem)
throws SQLException, AuthorizeException, IOException, SearchServiceException {
context.turnOffAuthorisationSystem();
workflowItem = context.reloadEntity(workflowItem);
workflowService.deleteWorkflowByWorkflowItem(context, workflowItem, admin);
context.commit();
indexer.commit();
context.restoreAuthSystemState();
}
private void returnClaimedTask(ClaimedTask taskToUnclaim) throws SQLException, IOException,
WorkflowConfigurationException, AuthorizeException, SearchServiceException {
final EPerson previousUser = context.getCurrentUser();
taskToUnclaim = context.reloadEntity(taskToUnclaim);
context.setCurrentUser(taskToUnclaim.getOwner());
XmlWorkflowItem workflowItem = taskToUnclaim.getWorkflowItem();
workflowService.deleteClaimedTask(context, workflowItem, taskToUnclaim);
workflowRequirementsService.removeClaimedUser(context, workflowItem, taskToUnclaim.getOwner(),
taskToUnclaim.getStepID());
context.commit();
indexer.commit();
context.setCurrentUser(previousUser);
}
private void claim(Workflow workflow, PoolTask task, EPerson user)
throws Exception {
final EPerson previousUser = context.getCurrentUser();
task = context.reloadEntity(task);
context.setCurrentUser(user);
Step step = workflow.getStep(task.getStepID());
WorkflowActionConfig currentActionConfig = step.getActionConfig(task.getActionID());
workflowService.doState(context, user, null, task.getWorkflowItem().getID(), workflow, currentActionConfig);
context.commit();
indexer.commit();
context.setCurrentUser(previousUser);
}
private void executeWorkflowAction(HttpServletRequest httpServletRequest, Workflow workflow, ClaimedTask task)
throws Exception {
final EPerson previousUser = context.getCurrentUser();
task = context.reloadEntity(task);
context.setCurrentUser(task.getOwner());
workflowService.doState(context, task.getOwner(), httpServletRequest, task.getWorkflowItem().getID(), workflow,
workflow.getStep(task.getStepID()).getActionConfig(task.getActionID()));
context.commit();
indexer.commit();
context.setCurrentUser(previousUser);
}
private void abort(XmlWorkflowItem workflowItem)
throws SQLException, AuthorizeException, IOException, SearchServiceException {
final EPerson previousUser = context.getCurrentUser();
workflowItem = context.reloadEntity(workflowItem);
context.setCurrentUser(admin);
workflowService.abort(context, workflowItem, admin);
context.commit();
indexer.commit();
context.setCurrentUser(previousUser);
}
}