From b8bca78e508d53f77cf4a49ac31a1eaaf0f78767 Mon Sep 17 00:00:00 2001 From: Ben Bosman Date: Tue, 18 Feb 2020 12:50:55 +0100 Subject: [PATCH 01/96] When moving metadata, make sure virtual metadata is handled correctly --- .../content/DSpaceObjectServiceImpl.java | 102 ++++++++++-------- .../org/dspace/content/ItemServiceImpl.java | 27 +++++ .../content/RelationshipMetadataValue.java | 6 ++ .../content/service/DSpaceObjectService.java | 31 +++--- 4 files changed, 107 insertions(+), 59 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/content/DSpaceObjectServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/DSpaceObjectServiceImpl.java index ee188dc144..2d55622e7b 100644 --- a/dspace-api/src/main/java/org/dspace/content/DSpaceObjectServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/DSpaceObjectServiceImpl.java @@ -207,8 +207,8 @@ public abstract class DSpaceObjectServiceImpl implements } @Override - public void addMetadata(Context context, T dso, String schema, String element, String qualifier, String lang, - List values) throws SQLException { + public List addMetadata(Context context, T dso, String schema, String element, String qualifier, + String lang, List values) throws SQLException { MetadataField metadataField = metadataFieldService.findByElement(context, schema, element, qualifier); if (metadataField == null) { throw new SQLException( @@ -216,12 +216,12 @@ public abstract class DSpaceObjectServiceImpl implements "exist!"); } - addMetadata(context, dso, metadataField, lang, values); + return addMetadata(context, dso, metadataField, lang, values); } @Override - public void addMetadata(Context context, T dso, String schema, String element, String qualifier, String lang, - List values, List authorities, List confidences) + public List addMetadata(Context context, T dso, String schema, String element, String qualifier, + String lang, List values, List authorities, List confidences) throws SQLException { // We will not verify that they are valid entries in the registry // until update() is called. @@ -231,15 +231,16 @@ public abstract class DSpaceObjectServiceImpl implements "bad_dublin_core schema=" + schema + "." + element + "." + qualifier + ". Metadata field does not " + "exist!"); } - addMetadata(context, dso, metadataField, lang, values, authorities, confidences); + return addMetadata(context, dso, metadataField, lang, values, authorities, confidences); } @Override - public void addMetadata(Context context, T dso, MetadataField metadataField, String lang, List values, - List authorities, List confidences) + public List addMetadata(Context context, T dso, MetadataField metadataField, String lang, + List values, List authorities, List confidences) throws SQLException { boolean authorityControlled = metadataAuthorityService.isAuthorityControlled(metadataField); boolean authorityRequired = metadataAuthorityService.isAuthorityRequired(metadataField); + List newMetadata = new ArrayList<>(values.size()); // We will not verify that they are valid entries in the registry // until update() is called. for (int i = 0; i < values.size(); i++) { @@ -250,6 +251,7 @@ public abstract class DSpaceObjectServiceImpl implements } } MetadataValue metadataValue = metadataValueService.create(context, dso, metadataField); + newMetadata.add(metadataValue); //Set place to list length of all metadatavalues for the given schema.element.qualifier combination. // Subtract one to adhere to the 0 as first element rule metadataValue.setPlace( @@ -304,29 +306,31 @@ public abstract class DSpaceObjectServiceImpl implements // metadataValueService.update(context, metadataValue); dso.addDetails(metadataField.toString()); } + return newMetadata; } @Override - public void addMetadata(Context context, T dso, MetadataField metadataField, String language, String value, - String authority, int confidence) throws SQLException { - addMetadata(context, dso, metadataField, language, Arrays.asList(value), Arrays.asList(authority), - Arrays.asList(confidence)); + public MetadataValue addMetadata(Context context, T dso, MetadataField metadataField, String language, + String value, String authority, int confidence) throws SQLException { + return addMetadata(context, dso, metadataField, language, Arrays.asList(value), Arrays.asList(authority), + Arrays.asList(confidence)).get(0); } @Override - public void addMetadata(Context context, T dso, String schema, String element, String qualifier, String lang, - String value) throws SQLException { - addMetadata(context, dso, schema, element, qualifier, lang, Arrays.asList(value)); + public MetadataValue addMetadata(Context context, T dso, String schema, String element, String qualifier, + String lang, String value) throws SQLException { + return addMetadata(context, dso, schema, element, qualifier, lang, Arrays.asList(value)).get(0); } @Override - public void addMetadata(Context context, T dso, MetadataField metadataField, String language, String value) + public MetadataValue addMetadata(Context context, T dso, MetadataField metadataField, String language, String value) throws SQLException { - addMetadata(context, dso, metadataField, language, Arrays.asList(value)); + return addMetadata(context, dso, metadataField, language, Arrays.asList(value)).get(0); } @Override - public void addMetadata(Context context, T dso, MetadataField metadataField, String language, List values) + public List addMetadata(Context context, T dso, MetadataField metadataField, String language, + List values) throws SQLException { if (metadataField != null) { String fieldKey = metadataAuthorityService @@ -343,18 +347,19 @@ public abstract class DSpaceObjectServiceImpl implements getAuthoritiesAndConfidences(fieldKey, null, values, authorities, confidences, i); } } - addMetadata(context, dso, metadataField, language, values, authorities, confidences); + return addMetadata(context, dso, metadataField, language, values, authorities, confidences); } else { - addMetadata(context, dso, metadataField, language, values, null, null); + return addMetadata(context, dso, metadataField, language, values, null, null); } } + return new ArrayList<>(0); } @Override - public void addMetadata(Context context, T dso, String schema, String element, String qualifier, String lang, - String value, String authority, int confidence) throws SQLException { - addMetadata(context, dso, schema, element, qualifier, lang, Arrays.asList(value), Arrays.asList(authority), - Arrays.asList(confidence)); + public MetadataValue addMetadata(Context context, T dso, String schema, String element, String qualifier, + String lang, String value, String authority, int confidence) throws SQLException { + return addMetadata(context, dso, schema, element, qualifier, lang, Arrays.asList(value), + Arrays.asList(authority), Arrays.asList(confidence)).get(0); } @Override @@ -660,33 +665,35 @@ public abstract class DSpaceObjectServiceImpl implements @Override public void addAndShiftRightMetadata(Context context, T dso, String schema, String element, String qualifier, String lang, String value, String authority, int confidence, int index) - throws SQLException { + throws SQLException { List list = getMetadata(dso, schema, element, qualifier); - clearMetadata(context, dso, schema, element, qualifier, Item.ANY); - int idx = 0; + int place = 0; boolean last = true; for (MetadataValue rr : list) { if (idx == index) { - addMetadata(context, dso, schema, element, qualifier, - lang, value, authority, confidence); + MetadataValue newMetadata = addMetadata(context, dso, schema, element, qualifier, + lang, value, authority, confidence); + + moveSingleMetadataValue(context, dso, schema, element, qualifier, place, newMetadata); + place++; last = false; } - addMetadata(context, dso, schema, element, qualifier, - rr.getLanguage(), rr.getValue(), rr.getAuthority(), rr.getConfidence()); + moveSingleMetadataValue(context, dso, schema, element, qualifier, place, rr); + place++; idx++; } if (last) { addMetadata(context, dso, schema, element, qualifier, - lang, value, authority, confidence); + lang, value, authority, confidence); } } @Override public void moveMetadata(Context context, T dso, String schema, String element, String qualifier, int from, int to) - throws SQLException, IllegalArgumentException { + throws SQLException, IllegalArgumentException { if (from == to) { throw new IllegalArgumentException("The \"from\" location MUST be different from \"to\" location"); @@ -696,11 +703,9 @@ public abstract class DSpaceObjectServiceImpl implements if (from >= list.size()) { throw new IllegalArgumentException( - "The \"from\" location MUST exist for the operation to be successful. Idx:" + from); + "The \"from\" location MUST exist for the operation to be successful. Idx:" + from); } - clearMetadata(context, dso, schema, element, qualifier, Item.ANY); - int idx = 0; MetadataValue moved = null; for (MetadataValue md : list) { @@ -712,30 +717,39 @@ public abstract class DSpaceObjectServiceImpl implements } idx = 0; + int place = 0; boolean last = true; for (MetadataValue rr : list) { if (idx == to && to < from) { - addMetadata(context, dso, schema, element, qualifier, moved.getLanguage(), moved.getValue(), - moved.getAuthority(), moved.getConfidence()); + moveSingleMetadataValue(context, dso, schema, element, qualifier, place, moved); + place++; last = false; } if (idx != from) { - addMetadata(context, dso, schema, element, qualifier, rr.getLanguage(), rr.getValue(), - rr.getAuthority(), rr.getConfidence()); + moveSingleMetadataValue(context, dso, schema, element, qualifier, place, rr); + place++; } if (idx == to && to > from) { - addMetadata(context, dso, schema, element, qualifier, moved.getLanguage(), moved.getValue(), - moved.getAuthority(), moved.getConfidence()); + moveSingleMetadataValue(context, dso, schema, element, qualifier, place, moved); + place++; last = false; } idx++; } if (last) { - addMetadata(context, dso, schema, element, qualifier, moved.getLanguage(), moved.getValue(), - moved.getAuthority(), moved.getConfidence()); + moveSingleMetadataValue(context, dso, schema, element, qualifier, place, moved); } } + /** + * Supports moving metadata by updating the place of the metadata value + */ + protected void moveSingleMetadataValue(Context context, T dso, String schema, String element, + String qualifier, int place, MetadataValue rr) throws SQLException { + //just move the metadata + rr.setPlace(place); + } + @Override public void replaceMetadata(Context context, T dso, String schema, String element, String qualifier, String lang, String value, String authority, int confidence, int index) throws SQLException { diff --git a/dspace-api/src/main/java/org/dspace/content/ItemServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/ItemServiceImpl.java index 9502a2ca32..ab302a3f9a 100644 --- a/dspace-api/src/main/java/org/dspace/content/ItemServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/ItemServiceImpl.java @@ -1372,6 +1372,33 @@ prevent the generation of resource policy entry values with null dspace_object a } + /** + * Supports moving metadata by adding the metadata value or updating the place of the relationship + */ + @Override + protected void moveSingleMetadataValue(Context context, Item dso, String schema, String element, + String qualifier, int place, MetadataValue rr) throws SQLException { + if (rr instanceof RelationshipMetadataValue) { + try { + //Retrieve the applicable relationship + Relationship rs = relationshipService.find(context, + ((RelationshipMetadataValue) rr).getRelationshipId()); + if (rs.getLeftItem() == dso) { + rs.setLeftPlace(place); + } else { + rs.setRightPlace(place); + } + relationshipService.update(context, rs); + } catch (Exception e) { + //should not occur, otherwise metadata can't be updated either + log.error("An error occurred while moving " + rr.getAuthority() + " for item " + dso.getID(), e); + } + } else { + //just move the metadata + rr.setPlace(place); + } + } + /** * This method will sort the List of MetadataValue objects based on the MetadataSchema, MetadataField Element, * MetadataField Qualifier and MetadataField Place in that order. diff --git a/dspace-api/src/main/java/org/dspace/content/RelationshipMetadataValue.java b/dspace-api/src/main/java/org/dspace/content/RelationshipMetadataValue.java index 88d2e38beb..f443f89ce5 100644 --- a/dspace-api/src/main/java/org/dspace/content/RelationshipMetadataValue.java +++ b/dspace-api/src/main/java/org/dspace/content/RelationshipMetadataValue.java @@ -7,6 +7,8 @@ */ package org.dspace.content; +import org.dspace.core.Constants; + /** * This class is used as a representation of MetadataValues for the MetadataValues that are derived from the * Relationships that the item has. This includes the useForPlace property which we'll have to use to determine @@ -57,4 +59,8 @@ public class RelationshipMetadataValue extends MetadataValue { } return super.equals(obj); } + + public int getRelationshipId() { + return Integer.parseInt(getAuthority().substring(Constants.VIRTUAL_AUTHORITY_PREFIX.length())); + } } diff --git a/dspace-api/src/main/java/org/dspace/content/service/DSpaceObjectService.java b/dspace-api/src/main/java/org/dspace/content/service/DSpaceObjectService.java index 753516a373..0fde8484d0 100644 --- a/dspace-api/src/main/java/org/dspace/content/service/DSpaceObjectService.java +++ b/dspace-api/src/main/java/org/dspace/content/service/DSpaceObjectService.java @@ -202,8 +202,8 @@ public interface DSpaceObjectService { * @param values the values to add. * @throws SQLException if database error */ - public void addMetadata(Context context, T dso, String schema, String element, String qualifier, String lang, - List values) throws SQLException; + public List addMetadata(Context context, T dso, String schema, String element, String qualifier, + String lang, List values) throws SQLException; /** * Add metadata fields. These are appended to existing values. @@ -225,8 +225,8 @@ public interface DSpaceObjectService { * @param confidences the authority confidence (default 0) * @throws SQLException if database error */ - public void addMetadata(Context context, T dso, String schema, String element, String qualifier, String lang, - List values, List authorities, List confidences) + public List addMetadata(Context context, T dso, String schema, String element, String qualifier, + String lang, List values, List authorities, List confidences) throws SQLException; /** @@ -244,9 +244,10 @@ public interface DSpaceObjectService { * @param authorities the external authority key for this value (or null) * @param confidences the authority confidence (default 0) * @throws SQLException if database error + * @return */ - public void addMetadata(Context context, T dso, MetadataField metadataField, String lang, List values, - List authorities, List confidences) throws SQLException; + public List addMetadata(Context context, T dso, MetadataField metadataField, String lang, + List values, List authorities, List confidences) throws SQLException; /** * Shortcut for {@link #addMetadata(Context, DSpaceObject, MetadataField, String, List, List, List)} when a single @@ -261,14 +262,14 @@ public interface DSpaceObjectService { * @param confidence * @throws SQLException */ - public void addMetadata(Context context, T dso, MetadataField metadataField, String language, String value, - String authority, int confidence) throws SQLException; + public MetadataValue addMetadata(Context context, T dso, MetadataField metadataField, String language, + String value, String authority, int confidence) throws SQLException; - public void addMetadata(Context context, T dso, MetadataField metadataField, String language, String value) + public MetadataValue addMetadata(Context context, T dso, MetadataField metadataField, String language, String value) throws SQLException; - public void addMetadata(Context context, T dso, MetadataField metadataField, String language, List values) - throws SQLException; + public List addMetadata(Context context, T dso, MetadataField metadataField, String language, + List values) throws SQLException; /** * Add a single metadata field. This is appended to existing @@ -287,8 +288,8 @@ public interface DSpaceObjectService { * @param value the value to add. * @throws SQLException if database error */ - public void addMetadata(Context context, T dso, String schema, String element, String qualifier, String lang, - String value) throws SQLException; + public MetadataValue addMetadata(Context context, T dso, String schema, String element, String qualifier, + String lang, String value) throws SQLException; /** * Add a single metadata field. This is appended to existing @@ -309,8 +310,8 @@ public interface DSpaceObjectService { * @param confidence the authority confidence (default 0) * @throws SQLException if database error */ - public void addMetadata(Context context, T dso, String schema, String element, String qualifier, String lang, - String value, String authority, int confidence) throws SQLException; + public MetadataValue addMetadata(Context context, T dso, String schema, String element, String qualifier, + String lang, String value, String authority, int confidence) throws SQLException; /** * Clear metadata values. As with getDC above, From 243b0c077fba8047a9718ac326ba426febcb9aca Mon Sep 17 00:00:00 2001 From: Ben Bosman Date: Tue, 18 Feb 2020 13:29:31 +0100 Subject: [PATCH 02/96] When moving metadata, make sure virtual metadata is handled correctly --- .../dspace/content/DSpaceObjectServiceImpl.java | 15 +++++++-------- .../java/org/dspace/content/ItemServiceImpl.java | 3 +-- .../impl/MetadataValueRemovePatchOperation.java | 14 ++------------ 3 files changed, 10 insertions(+), 22 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/content/DSpaceObjectServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/DSpaceObjectServiceImpl.java index 2d55622e7b..d990ea8a04 100644 --- a/dspace-api/src/main/java/org/dspace/content/DSpaceObjectServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/DSpaceObjectServiceImpl.java @@ -677,11 +677,11 @@ public abstract class DSpaceObjectServiceImpl implements MetadataValue newMetadata = addMetadata(context, dso, schema, element, qualifier, lang, value, authority, confidence); - moveSingleMetadataValue(context, dso, schema, element, qualifier, place, newMetadata); + moveSingleMetadataValue(context, dso, place, newMetadata); place++; last = false; } - moveSingleMetadataValue(context, dso, schema, element, qualifier, place, rr); + moveSingleMetadataValue(context, dso, place, rr); place++; idx++; } @@ -721,31 +721,30 @@ public abstract class DSpaceObjectServiceImpl implements boolean last = true; for (MetadataValue rr : list) { if (idx == to && to < from) { - moveSingleMetadataValue(context, dso, schema, element, qualifier, place, moved); + moveSingleMetadataValue(context, dso, place, moved); place++; last = false; } if (idx != from) { - moveSingleMetadataValue(context, dso, schema, element, qualifier, place, rr); + moveSingleMetadataValue(context, dso, place, rr); place++; } if (idx == to && to > from) { - moveSingleMetadataValue(context, dso, schema, element, qualifier, place, moved); + moveSingleMetadataValue(context, dso, place, moved); place++; last = false; } idx++; } if (last) { - moveSingleMetadataValue(context, dso, schema, element, qualifier, place, moved); + moveSingleMetadataValue(context, dso, place, moved); } } /** * Supports moving metadata by updating the place of the metadata value */ - protected void moveSingleMetadataValue(Context context, T dso, String schema, String element, - String qualifier, int place, MetadataValue rr) throws SQLException { + protected void moveSingleMetadataValue(Context context, T dso, int place, MetadataValue rr) { //just move the metadata rr.setPlace(place); } diff --git a/dspace-api/src/main/java/org/dspace/content/ItemServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/ItemServiceImpl.java index ab302a3f9a..00ab6df51e 100644 --- a/dspace-api/src/main/java/org/dspace/content/ItemServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/ItemServiceImpl.java @@ -1376,8 +1376,7 @@ prevent the generation of resource policy entry values with null dspace_object a * Supports moving metadata by adding the metadata value or updating the place of the relationship */ @Override - protected void moveSingleMetadataValue(Context context, Item dso, String schema, String element, - String qualifier, int place, MetadataValue rr) throws SQLException { + protected void moveSingleMetadataValue(Context context, Item dso, int place, MetadataValue rr) { if (rr instanceof RelationshipMetadataValue) { try { //Retrieve the applicable relationship diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/submit/factory/impl/MetadataValueRemovePatchOperation.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/submit/factory/impl/MetadataValueRemovePatchOperation.java index 5ea17cc3cd..4f2d092eb3 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/submit/factory/impl/MetadataValueRemovePatchOperation.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/submit/factory/impl/MetadataValueRemovePatchOperation.java @@ -8,6 +8,7 @@ package org.dspace.app.rest.submit.factory.impl; import java.sql.SQLException; +import java.util.Arrays; import java.util.List; import org.dspace.app.rest.model.MetadataValueRest; @@ -40,18 +41,7 @@ public abstract class MetadataValueRemovePatchOperation mm = getDSpaceObjectService().getMetadata(source, metadata[0], metadata[1], metadata[2], Item.ANY); - getDSpaceObjectService().clearMetadata(context, source, metadata[0], metadata[1], metadata[2], Item.ANY); - if (index != -1) { - int idx = 0; - for (MetadataValue m : mm) { - if (idx != index) { - getDSpaceObjectService().addMetadata(context, source, metadata[0], metadata[1], metadata[2], - m.getLanguage(), m.getValue(), m.getAuthority(), - m.getConfidence()); - } - idx++; - } - } + getDSpaceObjectService().removeMetadataValues(context, source, Arrays.asList(mm.get(index))); } protected abstract DSpaceObjectService getDSpaceObjectService(); From cc2534ddd3f3a045d9df4f6bd5cbea7c42a5cb93 Mon Sep 17 00:00:00 2001 From: Ben Bosman Date: Tue, 18 Feb 2020 14:41:48 +0100 Subject: [PATCH 03/96] When moving metadata, make sure virtual metadata is handled correctly --- .../factory/impl/MetadataValueRemovePatchOperation.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/submit/factory/impl/MetadataValueRemovePatchOperation.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/submit/factory/impl/MetadataValueRemovePatchOperation.java index 4f2d092eb3..1660a5455a 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/submit/factory/impl/MetadataValueRemovePatchOperation.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/submit/factory/impl/MetadataValueRemovePatchOperation.java @@ -41,7 +41,11 @@ public abstract class MetadataValueRemovePatchOperation mm = getDSpaceObjectService().getMetadata(source, metadata[0], metadata[1], metadata[2], Item.ANY); - getDSpaceObjectService().removeMetadataValues(context, source, Arrays.asList(mm.get(index))); + if (index != -1) { + getDSpaceObjectService().removeMetadataValues(context, source, Arrays.asList(mm.get(index))); + } else { + getDSpaceObjectService().clearMetadata(context, source, metadata[0], metadata[1], metadata[2], Item.ANY); + } } protected abstract DSpaceObjectService getDSpaceObjectService(); From 71c5cc678f9551a1f67f7e9adbfa42b9c86b6b14 Mon Sep 17 00:00:00 2001 From: Ben Bosman Date: Wed, 19 Feb 2020 14:50:26 +0100 Subject: [PATCH 04/96] When moving metadata, make sure virtual metadata is handled correctly --- .../dspace/content/DSpaceObjectServiceImpl.java | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/content/DSpaceObjectServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/DSpaceObjectServiceImpl.java index d990ea8a04..2751e2113f 100644 --- a/dspace-api/src/main/java/org/dspace/content/DSpaceObjectServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/DSpaceObjectServiceImpl.java @@ -755,19 +755,8 @@ public abstract class DSpaceObjectServiceImpl implements List list = getMetadata(dso, schema, element, qualifier); - clearMetadata(context, dso, schema, element, qualifier, Item.ANY); - - int idx = 0; - for (MetadataValue rr : list) { - if (idx == index) { - addMetadata(context, dso, schema, element, qualifier, - lang, value, authority, confidence); - } else { - addMetadata(context, dso, schema, element, qualifier, - rr.getLanguage(), rr.getValue(), rr.getAuthority(), rr.getConfidence()); - } - idx++; - } + removeMetadataValues(context, dso, Arrays.asList(list.get(index))); + addAndShiftRightMetadata(context, dso, schema, element, qualifier, lang, value, authority, confidence, index); } } From 42e0f2fb23eae1e1fbe27debb3bb188827e2243e Mon Sep 17 00:00:00 2001 From: Kristof De Langhe Date: Fri, 21 Feb 2020 14:18:54 +0100 Subject: [PATCH 05/96] 68820: Moving relationships in workspace items ITs --- .../org/dspace/app/rest/PatchMetadataIT.java | 292 ++++++++++++++++++ .../rest/builder/WorkspaceItemBuilder.java | 4 + 2 files changed, 296 insertions(+) create mode 100644 dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java new file mode 100644 index 0000000000..fc496b9453 --- /dev/null +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java @@ -0,0 +1,292 @@ +package org.dspace.app.rest; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.CoreMatchers.startsWith; +import static org.junit.Assert.assertThat; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.patch; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.util.ArrayList; +import java.util.List; + +import org.dspace.app.rest.builder.CollectionBuilder; +import org.dspace.app.rest.builder.CommunityBuilder; +import org.dspace.app.rest.builder.ItemBuilder; +import org.dspace.app.rest.builder.WorkspaceItemBuilder; +import org.dspace.app.rest.matcher.MetadataMatcher; +import org.dspace.app.rest.model.patch.MoveOperation; +import org.dspace.app.rest.model.patch.Operation; +import org.dspace.app.rest.test.AbstractEntityIntegrationTest; +import org.dspace.content.Collection; +import org.dspace.content.Community; +import org.dspace.content.Item; +import org.dspace.content.MetadataValue; +import org.dspace.content.RelationshipType; +import org.dspace.content.WorkspaceItem; +import org.dspace.content.service.EntityTypeService; +import org.dspace.content.service.ItemService; +import org.dspace.content.service.RelationshipTypeService; +import org.dspace.content.service.WorkspaceItemService; +import org.hamcrest.Matchers; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MvcResult; + +/** + * Created by kristof on 20/02/2020 + */ +public class PatchMetadataIT extends AbstractEntityIntegrationTest { + + @Autowired + private RelationshipTypeService relationshipTypeService; + + @Autowired + private EntityTypeService entityTypeService; + + @Autowired + private ItemService itemService; + + @Autowired + private WorkspaceItemService workspaceItemService; + + private Collection collection; + private WorkspaceItem publicationItem; + private Item personItem1; + private Item personItem2; + private RelationshipType publicationPersonRelationshipType; + + private List authorsOriginalOrder; + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + context.turnOffAuthorisationSystem(); + + Community community = CommunityBuilder.createCommunity(context) + .withName("Parent community") + .build(); + collection = CollectionBuilder.createCollection(context, community) + .withName("Collection") + .build(); + + context.restoreAuthSystemState(); + } + + private void initPersonPublicationWorkspace() throws Exception { + // Setup the original order of authors + authorsOriginalOrder = new ArrayList<>(); + authorsOriginalOrder.add("Whyte, William"); + // Second one will be virtual metadata + authorsOriginalOrder.add("Dahlen, Sarah"); + authorsOriginalOrder.add("Peterson, Karrie"); + authorsOriginalOrder.add("Perotti, Enrico"); + // 5th one will be virtual metadata + authorsOriginalOrder.add("Linton, Oliver"); + + context.turnOffAuthorisationSystem(); + + personItem1 = ItemBuilder.createItem(context, collection) + .withTitle("Person 1") + .withPersonIdentifierFirstName("Sarah") + .withPersonIdentifierLastName("Dahlen") + .withRelationshipType("Person") + .build(); + personItem2 = ItemBuilder.createItem(context, collection) + .withTitle("Person 2") + .withPersonIdentifierFirstName("Oliver") + .withPersonIdentifierLastName("Linton") + .withRelationshipType("Person") + .build(); + publicationItem = WorkspaceItemBuilder.createWorkspaceItem(context, collection) + .withTitle("Publication 1") + .withRelationshipType("Publication") + .build(); + publicationPersonRelationshipType = relationshipTypeService.findbyTypesAndTypeName(context, + entityTypeService.findByEntityType(context, "Publication"), + entityTypeService.findByEntityType(context, "Person"), + "isAuthorOfPublication", + "isPublicationOfAuthor"); + + String adminToken = getAuthToken(admin.getEmail(), password); + + // Make sure we grab the latest instance of the Item from the database before adding a regular author + WorkspaceItem publication = workspaceItemService.find(context, publicationItem.getID()); + itemService.addMetadata(context, publication.getItem(), + "dc", "contributor", "author", Item.ANY, authorsOriginalOrder.get(0)); + workspaceItemService.update(context, publication); + + context.restoreAuthSystemState(); + + // Create a relationship between publication and person 1 + MvcResult mvcResult = getClient(adminToken).perform(post("/api/core/relationships") + .param("relationshipType", publicationPersonRelationshipType.getID().toString()) + .contentType(MediaType.parseMediaType + (org.springframework.data.rest.webmvc.RestMediaTypes.TEXT_URI_LIST_VALUE)) + .content("https://localhost:8080/server/api/core/items/" + publicationItem.getItem().getID() + "\n" + + "https://localhost:8080/server/api/core/items/" + personItem1.getID())) + .andExpect(status().isCreated()) + .andReturn(); + + context.turnOffAuthorisationSystem(); + + // Add two more regular authors + List regularMetadata = new ArrayList<>(); + publication = workspaceItemService.find(context, publicationItem.getID()); + regularMetadata.add(authorsOriginalOrder.get(2)); + regularMetadata.add(authorsOriginalOrder.get(3)); + itemService.addMetadata(context, publication.getItem(), + "dc", "contributor", "author", null, regularMetadata); + workspaceItemService.update(context, publication); + + context.restoreAuthSystemState(); + + // Create a relationship between publication and person 2 + mvcResult = getClient(adminToken).perform(post("/api/core/relationships") + .param("relationshipType", publicationPersonRelationshipType.getID().toString()) + .contentType(MediaType.parseMediaType + (org.springframework.data.rest.webmvc.RestMediaTypes.TEXT_URI_LIST_VALUE)) + .content("https://localhost:8080/server/api/core/items/" + publicationItem.getItem().getID() + "\n" + + "https://localhost:8080/server/api/core/items/" + personItem2.getID())) + .andExpect(status().isCreated()) + .andReturn(); + + publication = workspaceItemService.find(context, publicationItem.getID()); + List publicationAuthorList = + itemService.getMetadata(publication.getItem(), "dc", "contributor", "author", Item.ANY); + assertThat(publicationAuthorList.size(), equalTo(5)); + assertThat(publicationAuthorList.get(0).getValue(), equalTo(authorsOriginalOrder.get(0))); + assertThat(publicationAuthorList.get(0).getAuthority(), not(startsWith("virtual::"))); + assertThat(publicationAuthorList.get(1).getValue(), equalTo(authorsOriginalOrder.get(1))); + assertThat(publicationAuthorList.get(1).getAuthority(), startsWith("virtual::")); + assertThat(publicationAuthorList.get(2).getValue(), equalTo(authorsOriginalOrder.get(2))); + assertThat(publicationAuthorList.get(2).getAuthority(), not(startsWith("virtual::"))); + assertThat(publicationAuthorList.get(3).getValue(), equalTo(authorsOriginalOrder.get(3))); + assertThat(publicationAuthorList.get(3).getAuthority(), not(startsWith("virtual::"))); + assertThat(publicationAuthorList.get(4).getValue(), equalTo(authorsOriginalOrder.get(4))); + assertThat(publicationAuthorList.get(4).getAuthority(), startsWith("virtual::")); + } + + @Test + public void moveTraditionalPageOneAuthorOneToZeroTest() throws Exception { + initPersonPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + moveTraditionalPageOneAuthorTest(1, 0, expectedOrder); + } + + @Test + public void moveTraditionalPageOneAuthorTwoToZeroTest() throws Exception { + initPersonPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + moveTraditionalPageOneAuthorTest(2, 0, expectedOrder); + } + + @Test + public void moveTraditionalPageOneAuthorThreeToZeroTest() throws Exception { + initPersonPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + moveTraditionalPageOneAuthorTest(3, 0, expectedOrder); + } + + @Test + public void moveTraditionalPageOneAuthorFourToZeroTest() throws Exception { + initPersonPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(4)); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(3)); + + moveTraditionalPageOneAuthorTest(4, 0, expectedOrder); + } + + @Test + public void moveTraditionalPageOneAuthorOneToThreeTest() throws Exception { + initPersonPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + moveTraditionalPageOneAuthorTest(1, 3, expectedOrder); + } + + @Test + public void moveTraditionalPageOneAuthorOneToFourTest() throws Exception { + initPersonPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(4)); + expectedOrder.add(authorsOriginalOrder.get(1)); + + moveTraditionalPageOneAuthorTest(1, 4, expectedOrder); + } + + private void moveTraditionalPageOneAuthorTest(int from, int path, List expectedOrder) throws Exception { + List ops = new ArrayList(); + MoveOperation moveOperation = getTraditionalPageOneMoveAuthorOperation(from, path); + ops.add(moveOperation); + String patchBody = getPatchContent(ops); + + String token = getAuthToken(admin.getEmail(), password); + + getClient(token).perform(patch("/api/submission/workspaceitems/" + publicationItem.getID()) + .content(patchBody) + .contentType(javax.ws.rs.core.MediaType.APPLICATION_JSON_PATCH_JSON)) + .andExpect(status().isOk()); + + String authorField = "dc.contributor.author"; + getClient().perform(get("/api/submission/workspaceitems/" + publicationItem.getID())) + .andExpect(status().isOk()) + .andExpect(content().contentType(contentType)) + .andExpect(jsonPath("$.sections.traditionalpageone", Matchers.allOf( + Matchers.is(MetadataMatcher.matchMetadata(authorField, expectedOrder.get(0), 0)), + Matchers.is(MetadataMatcher.matchMetadata(authorField, expectedOrder.get(1), 1)), + Matchers.is(MetadataMatcher.matchMetadata(authorField, expectedOrder.get(2), 2)), + Matchers.is(MetadataMatcher.matchMetadata(authorField, expectedOrder.get(3), 3)), + Matchers.is(MetadataMatcher.matchMetadata(authorField, expectedOrder.get(4), 4)) + ))); + } + + private MoveOperation getTraditionalPageOneMoveAuthorOperation(int from, int path) { + return new MoveOperation("/sections/traditionalpageone/dc.contributor.author/" + path, + "/sections/traditionalpageone/dc.contributor.author/" + from); + } + +} diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/builder/WorkspaceItemBuilder.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/builder/WorkspaceItemBuilder.java index 1d670bb3e1..eb5994dd32 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/builder/WorkspaceItemBuilder.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/builder/WorkspaceItemBuilder.java @@ -144,6 +144,10 @@ public class WorkspaceItemBuilder extends AbstractBuilder Date: Fri, 21 Feb 2020 17:27:25 +0100 Subject: [PATCH 06/96] 68820: PatchMetadataIT JavaDocs + header --- .../org/dspace/app/rest/PatchMetadataIT.java | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java index fc496b9453..0c86e02b05 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java @@ -1,3 +1,10 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ package org.dspace.app.rest; import static org.hamcrest.CoreMatchers.equalTo; @@ -80,6 +87,15 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { context.restoreAuthSystemState(); } + /** + * A method to create a workspace publication containing 5 authors: 3 regular authors and 2 related Person items. + * The authors are added in a specific order: + * - "Whyte, William": Regular author + * - "Dahlen, Sarah": Related Person + * - "Peterson, Karrie": Regular author + * - "Perotti, Enrico": Regular author + * - "Linton, Oliver": Related Person + */ private void initPersonPublicationWorkspace() throws Exception { // Setup the original order of authors authorsOriginalOrder = new ArrayList<>(); @@ -174,6 +190,12 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { assertThat(publicationAuthorList.get(4).getAuthority(), startsWith("virtual::")); } + /** + * This test will move an author (dc.description.author) within a workspace publication's "traditionalpageone" + * section from position 1 to 0 using a PATCH request and verify the order of the authors within the section. + * Original Order: 0,1,2,3,4 + * Expected Order: 1,0,2,3,4 + */ @Test public void moveTraditionalPageOneAuthorOneToZeroTest() throws Exception { initPersonPublicationWorkspace(); @@ -188,6 +210,12 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { moveTraditionalPageOneAuthorTest(1, 0, expectedOrder); } + /** + * This test will move an author (dc.description.author) within a workspace publication's "traditionalpageone" + * section from position 2 to 0 using a PATCH request and verify the order of the authors within the section. + * Original Order: 0,1,2,3,4 + * Expected Order: 2,0,1,3,4 + */ @Test public void moveTraditionalPageOneAuthorTwoToZeroTest() throws Exception { initPersonPublicationWorkspace(); @@ -202,6 +230,12 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { moveTraditionalPageOneAuthorTest(2, 0, expectedOrder); } + /** + * This test will move an author (dc.description.author) within a workspace publication's "traditionalpageone" + * section from position 3 to 0 using a PATCH request and verify the order of the authors within the section. + * Original Order: 0,1,2,3,4 + * Expected Order: 3,0,1,2,4 + */ @Test public void moveTraditionalPageOneAuthorThreeToZeroTest() throws Exception { initPersonPublicationWorkspace(); @@ -216,6 +250,12 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { moveTraditionalPageOneAuthorTest(3, 0, expectedOrder); } + /** + * This test will move an author (dc.description.author) within a workspace publication's "traditionalpageone" + * section from position 4 to 0 using a PATCH request and verify the order of the authors within the section. + * Original Order: 0,1,2,3,4 + * Expected Order: 4,0,1,2,3 + */ @Test public void moveTraditionalPageOneAuthorFourToZeroTest() throws Exception { initPersonPublicationWorkspace(); @@ -230,6 +270,12 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { moveTraditionalPageOneAuthorTest(4, 0, expectedOrder); } + /** + * This test will move an author (dc.description.author) within a workspace publication's "traditionalpageone" + * section from position 1 to 3 using a PATCH request and verify the order of the authors within the section. + * Original Order: 0,1,2,3,4 + * Expected Order: 0,2,3,1,4 + */ @Test public void moveTraditionalPageOneAuthorOneToThreeTest() throws Exception { initPersonPublicationWorkspace(); @@ -244,6 +290,12 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { moveTraditionalPageOneAuthorTest(1, 3, expectedOrder); } + /** + * This test will move an author (dc.description.author) within a workspace publication's "traditionalpageone" + * section from position 1 to 4 using a PATCH request and verify the order of the authors within the section. + * Original Order: 0,1,2,3,4 + * Expected Order: 0,2,3,4,1 + */ @Test public void moveTraditionalPageOneAuthorOneToFourTest() throws Exception { initPersonPublicationWorkspace(); @@ -258,6 +310,14 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { moveTraditionalPageOneAuthorTest(1, 4, expectedOrder); } + /** + * This method moves an author (dc.description.author) within a workspace publication's "traditionalpageone" + * section from position "from" to "path" using a PATCH request and verifies the order of the authors within the + * section using an ordered list of expected author names. + * @param from The "from" index to use for the Move operation + * @param path The "path" index to use for the Move operation + * @param expectedOrder A list of author names sorted in the expected order + */ private void moveTraditionalPageOneAuthorTest(int from, int path, List expectedOrder) throws Exception { List ops = new ArrayList(); MoveOperation moveOperation = getTraditionalPageOneMoveAuthorOperation(from, path); @@ -284,6 +344,12 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { ))); } + /** + * Create a move operation on a workspace item's "traditionalpageone" section for + * metadata field "dc.contributor.author". + * @param from The "from" index to use for the Move operation + * @param path The "path" index to use for the Move operation + */ private MoveOperation getTraditionalPageOneMoveAuthorOperation(int from, int path) { return new MoveOperation("/sections/traditionalpageone/dc.contributor.author/" + path, "/sections/traditionalpageone/dc.contributor.author/" + from); From 6748d63bba87a2c141cf94b800b73fdccca88df3 Mon Sep 17 00:00:00 2001 From: Yana De Pauw Date: Mon, 24 Feb 2020 12:52:07 +0100 Subject: [PATCH 07/96] 68850: Add tests for adding authors on different places on workspace item --- .../org/dspace/app/rest/PatchMetadataIT.java | 172 ++++++++++++++++++ 1 file changed, 172 insertions(+) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java index 0c86e02b05..231ee11538 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java @@ -26,6 +26,7 @@ import org.dspace.app.rest.builder.CommunityBuilder; import org.dspace.app.rest.builder.ItemBuilder; import org.dspace.app.rest.builder.WorkspaceItemBuilder; import org.dspace.app.rest.matcher.MetadataMatcher; +import org.dspace.app.rest.model.patch.AddOperation; import org.dspace.app.rest.model.patch.MoveOperation; import org.dspace.app.rest.model.patch.Operation; import org.dspace.app.rest.test.AbstractEntityIntegrationTest; @@ -71,6 +72,8 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { private List authorsOriginalOrder; + private String addedAuthor; + @Before @Override public void setUp() throws Exception { @@ -107,6 +110,8 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { // 5th one will be virtual metadata authorsOriginalOrder.add("Linton, Oliver"); + addedAuthor = "Semple, Robert"; + context.turnOffAuthorisationSystem(); personItem1 = ItemBuilder.createItem(context, collection) @@ -310,6 +315,138 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { moveTraditionalPageOneAuthorTest(1, 4, expectedOrder); } + /** + * This test will add an author (dc.description.author) within a workspace publication's "traditionalpageone" + * section at position 0 using a PATCH request and verify the place of the new author and the order of the + * authors within the section. + * Original Order: 0,1,2,3,4 + * Expected Order: +,0,1,2,3,4 (with + being the new author) + */ + public void addAuthorOnTraditionalPageOnePlaceZero() throws Exception { + initPersonPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(addedAuthor); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + addTraditionalPageOneAuthorTest("0", expectedOrder); + + } + + /** + * This test will add an author (dc.description.author) within a workspace publication's "traditionalpageone" + * section at position 1 using a PATCH request and verify the place of the new author and the order of the + * authors within the section. + * Original Order: 0,1,2,3,4 + * Expected Order: 0,+,1,2,3,4 (with + being the new author) + */ + public void addAuthorOnTraditionalPageOnePlaceOne() throws Exception { + initPersonPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(addedAuthor); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + addTraditionalPageOneAuthorTest("1", expectedOrder); + + } + + /** + * This test will add an author (dc.description.author) within a workspace publication's "traditionalpageone" + * section at position 2 using a PATCH request and verify the place of the new author and the order of the + * authors within the section. + * Original Order: 0,1,2,3,4 + * Expected Order: 0,1,+,2,3,4 (with + being the new author) + */ + public void addAuthorOnTraditionalPageOnePlaceTwo() throws Exception { + initPersonPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(addedAuthor); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + addTraditionalPageOneAuthorTest("2", expectedOrder); + + } + + /** + * This test will add an author (dc.description.author) within a workspace publication's "traditionalpageone" + * section at position 3 using a PATCH request and verify the place of the new author and the order of the + * authors within the section. + * Original Order: 0,1,2,3,4 + * Expected Order: 0,1,2,+,3,4 (with + being the new author) + */ + public void addAuthorOnTraditionalPageOnePlaceThree() throws Exception { + initPersonPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(addedAuthor); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + addTraditionalPageOneAuthorTest("3", expectedOrder); + + } + + /** + * This test will add an author (dc.description.author) within a workspace publication's "traditionalpageone" + * section at position 4 using a PATCH request and verify the place of the new author and the order of the + * authors within the section. + * Original Order: 0,1,2,3,4 + * Expected Order: 0,1,2,3,+,4 (with + being the new author) + */ + public void addAuthorOnTraditionalPageOnePlaceFour() throws Exception { + initPersonPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(addedAuthor); + expectedOrder.add(authorsOriginalOrder.get(4)); + + addTraditionalPageOneAuthorTest("4", expectedOrder); + + } + + /** + * This test will add an author (dc.description.author) within a workspace publication's "traditionalpageone" + * section at position 0 using a PATCH request and verify the place of the new author and the order of the + * authors within the section. + * Original Order: 0,1,2,3,4 + * Expected Order: +,0,1,2,3,4 (with + being the new author) + */ + public void addAuthorOnTraditionalPageOneLastPlace() throws Exception { + initPersonPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(4)); + expectedOrder.add(addedAuthor); + + addTraditionalPageOneAuthorTest("-", expectedOrder); + + } + /** * This method moves an author (dc.description.author) within a workspace publication's "traditionalpageone" * section from position "from" to "path" using a PATCH request and verifies the order of the authors within the @@ -344,6 +481,41 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { ))); } + /** + * This method adds an author (dc.description.author) within a workspace publication's "traditionalpageone" + * section to the position "path" using a PATCH request and verifies the place of the new author and the + * order of the previous authors within the section using an ordered list of expected author names. + * @param path The "path" index to use for the Add operation + * @param expectedOrder A list of author names sorted in the expected order + */ + private void addTraditionalPageOneAuthorTest(String path, List expectedOrder) throws Exception { + List ops = new ArrayList(); + AddOperation addOperation = new AddOperation("/sections/traditionalpageone/dc.contributor.author/" + path, + addedAuthor); + ops.add(addOperation); + String patchBody = getPatchContent(ops); + + String token = getAuthToken(admin.getEmail(), password); + + getClient(token).perform(patch("/api/submission/workspaceitems/" + publicationItem.getID()) + .content(patchBody) + .contentType(javax.ws.rs.core.MediaType.APPLICATION_JSON_PATCH_JSON)) + .andExpect(status().isOk()); + + String authorField = "dc.contributor.author"; + getClient().perform(get("/api/submission/workspaceitems/" + publicationItem.getID())) + .andExpect(status().isOk()) + .andExpect(content().contentType(contentType)) + .andExpect(jsonPath("$.sections.traditionalpageone", Matchers.allOf( + Matchers.is(MetadataMatcher.matchMetadata(authorField, expectedOrder.get(0), 0)), + Matchers.is(MetadataMatcher.matchMetadata(authorField, expectedOrder.get(1), 1)), + Matchers.is(MetadataMatcher.matchMetadata(authorField, expectedOrder.get(2), 2)), + Matchers.is(MetadataMatcher.matchMetadata(authorField, expectedOrder.get(3), 3)), + Matchers.is(MetadataMatcher.matchMetadata(authorField, expectedOrder.get(4), 4)), + Matchers.is(MetadataMatcher.matchMetadata(authorField, expectedOrder.get(5), 5)) + ))); + } + /** * Create a move operation on a workspace item's "traditionalpageone" section for * metadata field "dc.contributor.author". From 0691330d847d9659b8cc74f6f3e819e6f5aa5043 Mon Sep 17 00:00:00 2001 From: Yana De Pauw Date: Mon, 24 Feb 2020 15:23:53 +0100 Subject: [PATCH 08/96] 68851: Add test to Remove metadata through PATCH --- .../org/dspace/app/rest/PatchMetadataIT.java | 188 +++++++++++++++++- 1 file changed, 181 insertions(+), 7 deletions(-) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java index 231ee11538..7b76ea9a8f 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java @@ -26,9 +26,11 @@ import org.dspace.app.rest.builder.CommunityBuilder; import org.dspace.app.rest.builder.ItemBuilder; import org.dspace.app.rest.builder.WorkspaceItemBuilder; import org.dspace.app.rest.matcher.MetadataMatcher; +import org.dspace.app.rest.model.MetadataValueRest; import org.dspace.app.rest.model.patch.AddOperation; import org.dspace.app.rest.model.patch.MoveOperation; import org.dspace.app.rest.model.patch.Operation; +import org.dspace.app.rest.model.patch.RemoveOperation; import org.dspace.app.rest.test.AbstractEntityIntegrationTest; import org.dspace.content.Collection; import org.dspace.content.Community; @@ -322,7 +324,8 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { * Original Order: 0,1,2,3,4 * Expected Order: +,0,1,2,3,4 (with + being the new author) */ - public void addAuthorOnTraditionalPageOnePlaceZero() throws Exception { + @Test + public void addAuthorOnTraditionalPageOnePlaceZeroTest() throws Exception { initPersonPublicationWorkspace(); List expectedOrder = new ArrayList<>(); @@ -344,7 +347,8 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { * Original Order: 0,1,2,3,4 * Expected Order: 0,+,1,2,3,4 (with + being the new author) */ - public void addAuthorOnTraditionalPageOnePlaceOne() throws Exception { + @Test + public void addAuthorOnTraditionalPageOnePlaceOneTest() throws Exception { initPersonPublicationWorkspace(); List expectedOrder = new ArrayList<>(); @@ -366,7 +370,8 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { * Original Order: 0,1,2,3,4 * Expected Order: 0,1,+,2,3,4 (with + being the new author) */ - public void addAuthorOnTraditionalPageOnePlaceTwo() throws Exception { + @Test + public void addAuthorOnTraditionalPageOnePlaceTwoTest() throws Exception { initPersonPublicationWorkspace(); List expectedOrder = new ArrayList<>(); @@ -388,7 +393,8 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { * Original Order: 0,1,2,3,4 * Expected Order: 0,1,2,+,3,4 (with + being the new author) */ - public void addAuthorOnTraditionalPageOnePlaceThree() throws Exception { + @Test + public void addAuthorOnTraditionalPageOnePlaceThreeTest() throws Exception { initPersonPublicationWorkspace(); List expectedOrder = new ArrayList<>(); @@ -410,7 +416,8 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { * Original Order: 0,1,2,3,4 * Expected Order: 0,1,2,3,+,4 (with + being the new author) */ - public void addAuthorOnTraditionalPageOnePlaceFour() throws Exception { + @Test + public void addAuthorOnTraditionalPageOnePlaceFourTest() throws Exception { initPersonPublicationWorkspace(); List expectedOrder = new ArrayList<>(); @@ -432,7 +439,8 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { * Original Order: 0,1,2,3,4 * Expected Order: +,0,1,2,3,4 (with + being the new author) */ - public void addAuthorOnTraditionalPageOneLastPlace() throws Exception { + @Test + public void addAuthorOnTraditionalPageOneLastPlaceTest() throws Exception { initPersonPublicationWorkspace(); List expectedOrder = new ArrayList<>(); @@ -445,6 +453,137 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { addTraditionalPageOneAuthorTest("-", expectedOrder); + } + + /** + * This test will remove the author (dc.description.author) from a workspace publication's "traditionalpageone" + * section at position 0 using a PATCH request and verify the order of the remaining authors within the section. + * Original Order: 0,1,2,3,4 + * Expected Order: 1,2,3,4 + */ + @Test + public void removeAuthorOnTraditionalPageFromPlaceZeroTest() throws Exception { + initPersonPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + removeTraditionalPageOneAuthorTest(0, expectedOrder); + } + /** + * This test will remove the author (dc.description.author) from a workspace publication's "traditionalpageone" + * section at position 1 using a PATCH request and verify the order of the remaining authors within the section. + * Original Order: 0,1,2,3,4 + * Expected Order: 0,2,3,4 + */ + @Test + public void removeAuthorOnTraditionalPageFromPlaceOneTest() throws Exception { + initPersonPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + // The author at the first place is linked through a relationship and cannot be deleted through a PATCH request + removeTraditionalPageOneAuthorTest(1, expectedOrder); + } + /** + * This test will remove the author (dc.description.author) from a workspace publication's "traditionalpageone" + * section at position 2 using a PATCH request and verify the order of the remaining authors within the section. + * Original Order: 0,1,2,3,4 + * Expected Order: 0,1,3,4 + */ + @Test + public void removeAuthorOnTraditionalPageFromPlaceTwoTest() throws Exception { + initPersonPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + removeTraditionalPageOneAuthorTest(2, expectedOrder); + } + /** + * This test will remove the author (dc.description.author) from a workspace publication's "traditionalpageone" + * section at position 3 using a PATCH request and verify the order of the remaining authors within the section. + * Original Order: 0,1,2,3,4 + * Expected Order: 0,1,2,4 + */ + @Test + public void removeAuthorOnTraditionalPageFromPlaceThreeTest() throws Exception { + initPersonPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + removeTraditionalPageOneAuthorTest(3, expectedOrder); + } + /** + * This test will remove the author (dc.description.author) from a workspace publication's "traditionalpageone" + * section at position 4 using a PATCH request and verify the order of the remaining authors within the section. + * Original Order: 0,1,2,3,4 + * Expected Order: 0,1,2,3 + */ + @Test + public void removeAuthorOnTraditionalPageFromPlaceFourTest() throws Exception { + initPersonPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + // The author at the fourth place is linked through a relationship and cannot be deleted through a PATCH request + removeTraditionalPageOneAuthorTest(4, expectedOrder); + } + + /** + * This test will remove all authors (dc.description.author) from a workspace publication's "traditionalpageone" + * section using a PATCH request and verify that there are no remaining authors within the section. + */ + @Test + public void removeAllAuthorsOnTraditionalPageTest() throws Exception { + initPersonPublicationWorkspace(); + + List ops = new ArrayList(); + RemoveOperation removeOperation = new RemoveOperation("/sections/traditionalpageone/dc.contributor.author"); + ops.add(removeOperation); + String patchBody = getPatchContent(ops); + + String token = getAuthToken(admin.getEmail(), password); + + getClient(token).perform(patch("/api/submission/workspaceitems/" + publicationItem.getID()) + .content(patchBody) + .contentType(javax.ws.rs.core.MediaType.APPLICATION_JSON_PATCH_JSON)) + .andExpect(status().isOk()); + + String authorField = "dc.contributor.author"; + getClient().perform(get("/api/submission/workspaceitems/" + publicationItem.getID())) + .andExpect(status().isOk()) + .andExpect(content().contentType(contentType)) + .andExpect(jsonPath("$.sections.traditionalpageone", + // The author at the first and fourth place are linked through a relationship + // and cannot be deleted through a PATCH request + Matchers.allOf( + Matchers.is(MetadataMatcher.matchMetadata(authorField, authorsOriginalOrder.get(1), 0)), + Matchers.is(MetadataMatcher.matchMetadata(authorField, authorsOriginalOrder.get(4), 1)) + ))); + + + } /** @@ -490,8 +629,10 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { */ private void addTraditionalPageOneAuthorTest(String path, List expectedOrder) throws Exception { List ops = new ArrayList(); + MetadataValueRest value = new MetadataValueRest(); + value.setValue(addedAuthor); AddOperation addOperation = new AddOperation("/sections/traditionalpageone/dc.contributor.author/" + path, - addedAuthor); + value); ops.add(addOperation); String patchBody = getPatchContent(ops); @@ -516,6 +657,39 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { ))); } + /** + * This method removes an author (dc.description.author) within a workspace publication's "traditionalpageone" + * section from the position "path" using a PATCH request and verifies the order of the remaining authors + * within the section using an ordered list of expected author names. + * @param path The "path" index to use for the Remove operation + * @param expectedOrder A list of author names sorted in the expected order + */ + private void removeTraditionalPageOneAuthorTest(int path, List expectedOrder) throws Exception { + List ops = new ArrayList(); + RemoveOperation removeOperation = new RemoveOperation("/sections/traditionalpageone/dc.contributor.author/" + + path); + ops.add(removeOperation); + String patchBody = getPatchContent(ops); + + String token = getAuthToken(admin.getEmail(), password); + + getClient(token).perform(patch("/api/submission/workspaceitems/" + publicationItem.getID()) + .content(patchBody) + .contentType(javax.ws.rs.core.MediaType.APPLICATION_JSON_PATCH_JSON)) + .andExpect(status().isOk()); + + String authorField = "dc.contributor.author"; + getClient().perform(get("/api/submission/workspaceitems/" + publicationItem.getID())) + .andExpect(status().isOk()) + .andExpect(content().contentType(contentType)) + .andExpect(jsonPath("$.sections.traditionalpageone", Matchers.allOf( + Matchers.is(MetadataMatcher.matchMetadata(authorField, expectedOrder.get(0), 0)), + Matchers.is(MetadataMatcher.matchMetadata(authorField, expectedOrder.get(1), 1)), + Matchers.is(MetadataMatcher.matchMetadata(authorField, expectedOrder.get(2), 2)), + Matchers.is(MetadataMatcher.matchMetadata(authorField, expectedOrder.get(3), 3)) + ))); + } + /** * Create a move operation on a workspace item's "traditionalpageone" section for * metadata field "dc.contributor.author". From af2f1ee15b8a6b7b338d8f372c58649c5bf92933 Mon Sep 17 00:00:00 2001 From: Yana De Pauw Date: Tue, 25 Feb 2020 11:30:43 +0100 Subject: [PATCH 09/96] 68919: Add replace tests --- .../org/dspace/app/rest/PatchMetadataIT.java | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java index 7b76ea9a8f..9508782b39 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java @@ -31,6 +31,7 @@ import org.dspace.app.rest.model.patch.AddOperation; import org.dspace.app.rest.model.patch.MoveOperation; import org.dspace.app.rest.model.patch.Operation; import org.dspace.app.rest.model.patch.RemoveOperation; +import org.dspace.app.rest.model.patch.ReplaceOperation; import org.dspace.app.rest.test.AbstractEntityIntegrationTest; import org.dspace.content.Collection; import org.dspace.content.Community; @@ -75,6 +76,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { private List authorsOriginalOrder; private String addedAuthor; + private String replacedAuthor; @Before @Override @@ -113,6 +115,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { authorsOriginalOrder.add("Linton, Oliver"); addedAuthor = "Semple, Robert"; + replacedAuthor = "New Value"; context.turnOffAuthorisationSystem(); @@ -317,6 +320,64 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { moveTraditionalPageOneAuthorTest(1, 4, expectedOrder); } + /** + * This test will replace an author (dc.description.author) within a workspace publication's "traditionalpageone" + * section at position 0 using a PATCH request and verify the order and value of the authors within the section. + * @throws Exception + */ + @Test + public void replaceTraditionalPageOneAuthorZeroTest() throws Exception { + initPersonPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(replacedAuthor); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + replaceTraditionalPageOneAuthorTest(0, expectedOrder); + } + + /** + * This test will replace an author (dc.description.author) within a workspace publication's "traditionalpageone" + * section at position 2 using a PATCH request and verify the order and value of the authors within the section. + * @throws Exception + */ + @Test + public void replaceTraditionalPageOneAuthorTwoTest() throws Exception { + initPersonPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(replacedAuthor); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + replaceTraditionalPageOneAuthorTest(2, expectedOrder); + } + + /** + * This test will replace an author (dc.description.author) within a workspace publication's "traditionalpageone" + * section at position 3 using a PATCH request and verify the order and value of the authors within the section. + * @throws Exception + */ + @Test + public void replaceTraditionalPageOneAuthorThreeTest() throws Exception { + initPersonPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(replacedAuthor); + expectedOrder.add(authorsOriginalOrder.get(4)); + + replaceTraditionalPageOneAuthorTest(3, expectedOrder); + } + + /** * This test will add an author (dc.description.author) within a workspace publication's "traditionalpageone" * section at position 0 using a PATCH request and verify the place of the new author and the order of the @@ -620,6 +681,43 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { ))); } + /** + * This method replaces an author (dc.description.author) within a workspace publication's "traditionalpageone" + * section at position "path" using a PATCH request and verifies the order of the authors within the + * section using an ordered list of expected author names. + * @param path The "path" index to use for the Replace operation + * @param expectedOrder A list of author names sorted in the expected order + */ + private void replaceTraditionalPageOneAuthorTest(int path, List expectedOrder) throws Exception { + List ops = new ArrayList(); + MetadataValueRest value = new MetadataValueRest(); + value.setValue(replacedAuthor); + + ReplaceOperation replaceOperation = new ReplaceOperation("/sections/traditionalpageone/dc.contributor.author/" + + path, value); + ops.add(replaceOperation); + String patchBody = getPatchContent(ops); + + String token = getAuthToken(admin.getEmail(), password); + + getClient(token).perform(patch("/api/submission/workspaceitems/" + publicationItem.getID()) + .content(patchBody) + .contentType(javax.ws.rs.core.MediaType.APPLICATION_JSON_PATCH_JSON)) + .andExpect(status().isOk()); + + String authorField = "dc.contributor.author"; + getClient().perform(get("/api/submission/workspaceitems/" + publicationItem.getID())) + .andExpect(status().isOk()) + .andExpect(content().contentType(contentType)) + .andExpect(jsonPath("$.sections.traditionalpageone", Matchers.allOf( + Matchers.is(MetadataMatcher.matchMetadata(authorField, expectedOrder.get(0), 0)), + Matchers.is(MetadataMatcher.matchMetadata(authorField, expectedOrder.get(1), 1)), + Matchers.is(MetadataMatcher.matchMetadata(authorField, expectedOrder.get(2), 2)), + Matchers.is(MetadataMatcher.matchMetadata(authorField, expectedOrder.get(3), 3)), + Matchers.is(MetadataMatcher.matchMetadata(authorField, expectedOrder.get(4), 4)) + ))); + } + /** * This method adds an author (dc.description.author) within a workspace publication's "traditionalpageone" * section to the position "path" using a PATCH request and verifies the place of the new author and the From 9b442743b69ca91aed952d54f79a80003cb280be Mon Sep 17 00:00:00 2001 From: Yana De Pauw Date: Tue, 25 Feb 2020 13:32:28 +0100 Subject: [PATCH 10/96] 69108: Fix comments --- .../org/dspace/app/rest/PatchMetadataIT.java | 57 ++++++++++--------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java index 9508782b39..1959079a84 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java @@ -201,7 +201,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { } /** - * This test will move an author (dc.description.author) within a workspace publication's "traditionalpageone" + * This test will move an author (dc.contributor.author) within a workspace publication's "traditionalpageone" * section from position 1 to 0 using a PATCH request and verify the order of the authors within the section. * Original Order: 0,1,2,3,4 * Expected Order: 1,0,2,3,4 @@ -221,7 +221,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { } /** - * This test will move an author (dc.description.author) within a workspace publication's "traditionalpageone" + * This test will move an author (dc.contributor.author) within a workspace publication's "traditionalpageone" * section from position 2 to 0 using a PATCH request and verify the order of the authors within the section. * Original Order: 0,1,2,3,4 * Expected Order: 2,0,1,3,4 @@ -241,7 +241,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { } /** - * This test will move an author (dc.description.author) within a workspace publication's "traditionalpageone" + * This test will move an author (dc.contributor.author) within a workspace publication's "traditionalpageone" * section from position 3 to 0 using a PATCH request and verify the order of the authors within the section. * Original Order: 0,1,2,3,4 * Expected Order: 3,0,1,2,4 @@ -261,7 +261,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { } /** - * This test will move an author (dc.description.author) within a workspace publication's "traditionalpageone" + * This test will move an author (dc.contributor.author) within a workspace publication's "traditionalpageone" * section from position 4 to 0 using a PATCH request and verify the order of the authors within the section. * Original Order: 0,1,2,3,4 * Expected Order: 4,0,1,2,3 @@ -281,7 +281,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { } /** - * This test will move an author (dc.description.author) within a workspace publication's "traditionalpageone" + * This test will move an author (dc.contributor.author) within a workspace publication's "traditionalpageone" * section from position 1 to 3 using a PATCH request and verify the order of the authors within the section. * Original Order: 0,1,2,3,4 * Expected Order: 0,2,3,1,4 @@ -301,7 +301,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { } /** - * This test will move an author (dc.description.author) within a workspace publication's "traditionalpageone" + * This test will move an author (dc.contributor.author) within a workspace publication's "traditionalpageone" * section from position 1 to 4 using a PATCH request and verify the order of the authors within the section. * Original Order: 0,1,2,3,4 * Expected Order: 0,2,3,4,1 @@ -321,7 +321,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { } /** - * This test will replace an author (dc.description.author) within a workspace publication's "traditionalpageone" + * This test will replace an author (dc.contributor.author) within a workspace publication's "traditionalpageone" * section at position 0 using a PATCH request and verify the order and value of the authors within the section. * @throws Exception */ @@ -340,7 +340,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { } /** - * This test will replace an author (dc.description.author) within a workspace publication's "traditionalpageone" + * This test will replace an author (dc.contributor.author) within a workspace publication's "traditionalpageone" * section at position 2 using a PATCH request and verify the order and value of the authors within the section. * @throws Exception */ @@ -359,7 +359,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { } /** - * This test will replace an author (dc.description.author) within a workspace publication's "traditionalpageone" + * This test will replace an author (dc.contributor.author) within a workspace publication's "traditionalpageone" * section at position 3 using a PATCH request and verify the order and value of the authors within the section. * @throws Exception */ @@ -379,7 +379,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { /** - * This test will add an author (dc.description.author) within a workspace publication's "traditionalpageone" + * This test will add an author (dc.contributor.author) within a workspace publication's "traditionalpageone" * section at position 0 using a PATCH request and verify the place of the new author and the order of the * authors within the section. * Original Order: 0,1,2,3,4 @@ -402,7 +402,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { } /** - * This test will add an author (dc.description.author) within a workspace publication's "traditionalpageone" + * This test will add an author (dc.contributor.author) within a workspace publication's "traditionalpageone" * section at position 1 using a PATCH request and verify the place of the new author and the order of the * authors within the section. * Original Order: 0,1,2,3,4 @@ -425,7 +425,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { } /** - * This test will add an author (dc.description.author) within a workspace publication's "traditionalpageone" + * This test will add an author (dc.contributor.author) within a workspace publication's "traditionalpageone" * section at position 2 using a PATCH request and verify the place of the new author and the order of the * authors within the section. * Original Order: 0,1,2,3,4 @@ -448,7 +448,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { } /** - * This test will add an author (dc.description.author) within a workspace publication's "traditionalpageone" + * This test will add an author (dc.contributor.author) within a workspace publication's "traditionalpageone" * section at position 3 using a PATCH request and verify the place of the new author and the order of the * authors within the section. * Original Order: 0,1,2,3,4 @@ -471,7 +471,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { } /** - * This test will add an author (dc.description.author) within a workspace publication's "traditionalpageone" + * This test will add an author (dc.contributor.author) within a workspace publication's "traditionalpageone" * section at position 4 using a PATCH request and verify the place of the new author and the order of the * authors within the section. * Original Order: 0,1,2,3,4 @@ -494,7 +494,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { } /** - * This test will add an author (dc.description.author) within a workspace publication's "traditionalpageone" + * This test will add an author (dc.contributor.author) within a workspace publication's "traditionalpageone" * section at position 0 using a PATCH request and verify the place of the new author and the order of the * authors within the section. * Original Order: 0,1,2,3,4 @@ -517,7 +517,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { } /** - * This test will remove the author (dc.description.author) from a workspace publication's "traditionalpageone" + * This test will remove the author (dc.contributor.author) from a workspace publication's "traditionalpageone" * section at position 0 using a PATCH request and verify the order of the remaining authors within the section. * Original Order: 0,1,2,3,4 * Expected Order: 1,2,3,4 @@ -535,10 +535,10 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { removeTraditionalPageOneAuthorTest(0, expectedOrder); } /** - * This test will remove the author (dc.description.author) from a workspace publication's "traditionalpageone" + * This test will remove the author (dc.contributor.author) from a workspace publication's "traditionalpageone" * section at position 1 using a PATCH request and verify the order of the remaining authors within the section. * Original Order: 0,1,2,3,4 - * Expected Order: 0,2,3,4 + * Expected Order: 0,1,2,3,4 */ @Test public void removeAuthorOnTraditionalPageFromPlaceOneTest() throws Exception { @@ -555,7 +555,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { removeTraditionalPageOneAuthorTest(1, expectedOrder); } /** - * This test will remove the author (dc.description.author) from a workspace publication's "traditionalpageone" + * This test will remove the author (dc.contributor.author) from a workspace publication's "traditionalpageone" * section at position 2 using a PATCH request and verify the order of the remaining authors within the section. * Original Order: 0,1,2,3,4 * Expected Order: 0,1,3,4 @@ -573,7 +573,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { removeTraditionalPageOneAuthorTest(2, expectedOrder); } /** - * This test will remove the author (dc.description.author) from a workspace publication's "traditionalpageone" + * This test will remove the author (dc.contributor.author) from a workspace publication's "traditionalpageone" * section at position 3 using a PATCH request and verify the order of the remaining authors within the section. * Original Order: 0,1,2,3,4 * Expected Order: 0,1,2,4 @@ -591,10 +591,10 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { removeTraditionalPageOneAuthorTest(3, expectedOrder); } /** - * This test will remove the author (dc.description.author) from a workspace publication's "traditionalpageone" + * This test will remove the author (dc.contributor.author) from a workspace publication's "traditionalpageone" * section at position 4 using a PATCH request and verify the order of the remaining authors within the section. * Original Order: 0,1,2,3,4 - * Expected Order: 0,1,2,3 + * Expected Order: 0,1,2,3,4 */ @Test public void removeAuthorOnTraditionalPageFromPlaceFourTest() throws Exception { @@ -612,8 +612,9 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { } /** - * This test will remove all authors (dc.description.author) from a workspace publication's "traditionalpageone" - * section using a PATCH request and verify that there are no remaining authors within the section. + * This test will remove all authors (dc.contributor.author) that are not linked through a relationship from a + * workspace publication's "traditionalpageone" section using a PATCH request and verify that the only remaining + * authors are those coming from a relationship. */ @Test public void removeAllAuthorsOnTraditionalPageTest() throws Exception { @@ -648,7 +649,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { } /** - * This method moves an author (dc.description.author) within a workspace publication's "traditionalpageone" + * This method moves an author (dc.contributor.author) within a workspace publication's "traditionalpageone" * section from position "from" to "path" using a PATCH request and verifies the order of the authors within the * section using an ordered list of expected author names. * @param from The "from" index to use for the Move operation @@ -682,7 +683,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { } /** - * This method replaces an author (dc.description.author) within a workspace publication's "traditionalpageone" + * This method replaces an author (dc.contributor.author) within a workspace publication's "traditionalpageone" * section at position "path" using a PATCH request and verifies the order of the authors within the * section using an ordered list of expected author names. * @param path The "path" index to use for the Replace operation @@ -719,7 +720,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { } /** - * This method adds an author (dc.description.author) within a workspace publication's "traditionalpageone" + * This method adds an author (dc.contributor.author) within a workspace publication's "traditionalpageone" * section to the position "path" using a PATCH request and verifies the place of the new author and the * order of the previous authors within the section using an ordered list of expected author names. * @param path The "path" index to use for the Add operation @@ -756,7 +757,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { } /** - * This method removes an author (dc.description.author) within a workspace publication's "traditionalpageone" + * This method removes an author (dc.contributor.author) within a workspace publication's "traditionalpageone" * section from the position "path" using a PATCH request and verifies the order of the remaining authors * within the section using an ordered list of expected author names. * @param path The "path" index to use for the Remove operation From d4cc5f4587f12c7bbcce4b6dc982bf49ea2b9eb4 Mon Sep 17 00:00:00 2001 From: Ben Bosman Date: Tue, 17 Mar 2020 16:41:10 +0100 Subject: [PATCH 11/96] Using token since workspace security is now in place --- .../test/java/org/dspace/app/rest/PatchMetadataIT.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java index 1959079a84..eb6b42d246 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java @@ -670,7 +670,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { .andExpect(status().isOk()); String authorField = "dc.contributor.author"; - getClient().perform(get("/api/submission/workspaceitems/" + publicationItem.getID())) + getClient(token).perform(get("/api/submission/workspaceitems/" + publicationItem.getID())) .andExpect(status().isOk()) .andExpect(content().contentType(contentType)) .andExpect(jsonPath("$.sections.traditionalpageone", Matchers.allOf( @@ -707,7 +707,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { .andExpect(status().isOk()); String authorField = "dc.contributor.author"; - getClient().perform(get("/api/submission/workspaceitems/" + publicationItem.getID())) + getClient(token).perform(get("/api/submission/workspaceitems/" + publicationItem.getID())) .andExpect(status().isOk()) .andExpect(content().contentType(contentType)) .andExpect(jsonPath("$.sections.traditionalpageone", Matchers.allOf( @@ -743,7 +743,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { .andExpect(status().isOk()); String authorField = "dc.contributor.author"; - getClient().perform(get("/api/submission/workspaceitems/" + publicationItem.getID())) + getClient(token).perform(get("/api/submission/workspaceitems/" + publicationItem.getID())) .andExpect(status().isOk()) .andExpect(content().contentType(contentType)) .andExpect(jsonPath("$.sections.traditionalpageone", Matchers.allOf( @@ -778,7 +778,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { .andExpect(status().isOk()); String authorField = "dc.contributor.author"; - getClient().perform(get("/api/submission/workspaceitems/" + publicationItem.getID())) + getClient(token).perform(get("/api/submission/workspaceitems/" + publicationItem.getID())) .andExpect(status().isOk()) .andExpect(content().contentType(contentType)) .andExpect(jsonPath("$.sections.traditionalpageone", Matchers.allOf( From 932664650c696fd037ddd2d01a2f97eeb9106a4b Mon Sep 17 00:00:00 2001 From: Ben Bosman Date: Tue, 17 Mar 2020 17:24:11 +0100 Subject: [PATCH 12/96] Using token since workspace security is now in place --- .../src/test/java/org/dspace/app/rest/PatchMetadataIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java index eb6b42d246..e1b7258ae1 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java @@ -633,7 +633,7 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { .andExpect(status().isOk()); String authorField = "dc.contributor.author"; - getClient().perform(get("/api/submission/workspaceitems/" + publicationItem.getID())) + getClient(token).perform(get("/api/submission/workspaceitems/" + publicationItem.getID())) .andExpect(status().isOk()) .andExpect(content().contentType(contentType)) .andExpect(jsonPath("$.sections.traditionalpageone", From 5c8ff2c74312c5c97a14c01e2efa4d8521c1c1cb Mon Sep 17 00:00:00 2001 From: Andrea Bollini Date: Sat, 20 Jun 2020 18:49:56 +0200 Subject: [PATCH 13/96] DS-4530 enforce the use of the latest override of the findOne when checking permission for embedding --- .../app/rest/converter/ConverterService.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/ConverterService.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/ConverterService.java index fc786bfc85..563f2045ca 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/ConverterService.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/ConverterService.java @@ -149,14 +149,29 @@ public class ConverterService { DSpaceRestRepository repositoryToUse = utils .getResourceRepositoryByCategoryAndModel(baseObjectRest.getCategory(), baseObjectRest.getType()); Annotation preAuthorize = null; + int maxDepth = 0; for (Method m : repositoryToUse.getClass().getMethods()) { if (StringUtils.equalsIgnoreCase(m.getName(), "findOne")) { - preAuthorize = AnnotationUtils.findAnnotation(m, PreAuthorize.class); + int depth = howManySuperclass(m.getDeclaringClass()); + if (depth > maxDepth) { + preAuthorize = AnnotationUtils.findAnnotation(m, PreAuthorize.class); + maxDepth = depth; + } } } return preAuthorize; } + private int howManySuperclass(Class declaringClass) { + Class curr = declaringClass; + int count = 0; + while (curr != Object.class) { + curr = curr.getSuperclass(); + count++; + } + return count; + } + private Annotation getDefaultFindOnePreAuthorize() { for (Method m : DSpaceRestRepository.class.getMethods()) { if (StringUtils.equalsIgnoreCase(m.getName(), "findOne")) { From f50d0d664832142525d68ed22ae4e5ec935d9770 Mon Sep 17 00:00:00 2001 From: Andrea Bollini Date: Sat, 20 Jun 2020 18:50:36 +0200 Subject: [PATCH 14/96] Add dependency needed by the pubmed live import provider --- dspace-api/pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dspace-api/pom.xml b/dspace-api/pom.xml index a0714c04ee..7de2b782af 100644 --- a/dspace-api/pom.xml +++ b/dspace-api/pom.xml @@ -622,6 +622,12 @@ jersey-client ${jersey.version} + + + org.glassfish.jersey.inject + jersey-hk2 + ${jersey.version} + com.amazonaws From 38c566723384729f1e97ab0b7687d6696bb79180 Mon Sep 17 00:00:00 2001 From: Andrea Bollini Date: Sat, 20 Jun 2020 18:52:23 +0200 Subject: [PATCH 15/96] DS-4515 enable live import provider as external authority --- .../provider/impl/LiveImportDataProvider.java | 142 ++++++++++++++++++ .../config/spring/api/external-services.xml | 6 + 2 files changed, 148 insertions(+) create mode 100644 dspace-api/src/main/java/org/dspace/external/provider/impl/LiveImportDataProvider.java diff --git a/dspace-api/src/main/java/org/dspace/external/provider/impl/LiveImportDataProvider.java b/dspace-api/src/main/java/org/dspace/external/provider/impl/LiveImportDataProvider.java new file mode 100644 index 0000000000..776984ddb7 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/external/provider/impl/LiveImportDataProvider.java @@ -0,0 +1,142 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.external.provider.impl; + +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; +import org.dspace.content.dto.MetadataValueDTO; +import org.dspace.external.model.ExternalDataObject; +import org.dspace.external.provider.ExternalDataProvider; +import org.dspace.importer.external.datamodel.ImportRecord; +import org.dspace.importer.external.exception.MetadataSourceException; +import org.dspace.importer.external.metadatamapping.MetadatumDTO; +import org.dspace.importer.external.service.components.MetadataSource; + +/** + * This class allows to configure a Live Import Provider as an External Data Provider + * + * @author Andrea Bollini (andrea.bollini at 4science.it) + * + */ +public class LiveImportDataProvider implements ExternalDataProvider { + /** + * The {@link MetadataSource} live import provider + */ + private MetadataSource metadataSource; + + /** + * An unique human readable identifier for this provider + */ + private String sourceIdentifier; + + private String recordIdMetadata; + + private String displayMetadata = "dc.title"; + + @Override + public String getSourceIdentifier() { + return sourceIdentifier; + } + + public void setSourceIdentifier(String sourceIdentifier) { + this.sourceIdentifier = sourceIdentifier; + } + + public void setMetadataSource(MetadataSource metadataSource) { + this.metadataSource = metadataSource; + } + + public void setRecordIdMetadata(String recordIdMetadata) { + this.recordIdMetadata = recordIdMetadata; + } + + public void setDisplayMetadata(String displayMetadata) { + this.displayMetadata = displayMetadata; + } + + @Override + public Optional getExternalDataObject(String id) { + try { + ExternalDataObject externalDataObject = getExternalDataObject(metadataSource.getRecord(id)); + return Optional.of(externalDataObject); + } catch (MetadataSourceException e) { + throw new RuntimeException( + "The live import provider " + metadataSource.getImportSource() + " throws an exception", e); + } + } + + @Override + public List searchExternalDataObjects(String query, int start, int limit) { + Collection records; + try { + records = metadataSource.getRecords(query, start, limit); + return records.stream().map(r -> getExternalDataObject(r)).collect(Collectors.toList()); + } catch (MetadataSourceException e) { + throw new RuntimeException( + "The live import provider " + metadataSource.getImportSource() + " throws an exception", e); + } + } + + @Override + public boolean supports(String source) { + return StringUtils.equalsIgnoreCase(sourceIdentifier, source); + } + + @Override + public int getNumberOfResults(String query) { + try { + return metadataSource.getNbRecords(query); + } catch (MetadataSourceException e) { + throw new RuntimeException( + "The live import provider " + metadataSource.getImportSource() + " throws an exception", e); + } + } + + /** + * Internal method to convert an ImportRecord to an ExternalDataObject + * + * FIXME it would be useful to remove ImportRecord at all in favor of the + * ExternalDataObject + * + * @param record + * @return + */ + private ExternalDataObject getExternalDataObject(ImportRecord record) { + ExternalDataObject externalDataObject = new ExternalDataObject(sourceIdentifier); + String id = getFirstValue(record, recordIdMetadata); + String display = getFirstValue(record, displayMetadata); + externalDataObject.setId(id); + externalDataObject.setDisplayValue(display); + externalDataObject.setValue(display); + for (MetadatumDTO dto : record.getValueList()) { + // FIXME it would be useful to remove MetadatumDTO in favor of MetadataValueDTO + MetadataValueDTO mvDTO = new MetadataValueDTO(); + mvDTO.setSchema(dto.getSchema()); + mvDTO.setElement(dto.getElement()); + mvDTO.setQualifier(dto.getQualifier()); + mvDTO.setValue(dto.getValue()); + externalDataObject.addMetadata(mvDTO); + } + return externalDataObject; + } + + private String getFirstValue(ImportRecord record, String metadata) { + String id = null; + String[] split = StringUtils.split(metadata, ".", 3); + Collection values = record.getValue(split[0], split[1], split.length == 3 ? split[2] : null); + if (!values.isEmpty()) { + id = (values.iterator().next().getValue()); + } + return id; + } + +} diff --git a/dspace/config/spring/api/external-services.xml b/dspace/config/spring/api/external-services.xml index 520c21a963..098b53c2ca 100644 --- a/dspace/config/spring/api/external-services.xml +++ b/dspace/config/spring/api/external-services.xml @@ -30,5 +30,11 @@ + + + + + + From bd2cf94376e5560559d838c04d74cd0817c654a4 Mon Sep 17 00:00:00 2001 From: Andrea Bollini Date: Sat, 20 Jun 2020 18:52:45 +0200 Subject: [PATCH 16/96] DS-4529 External authorities endpoint doesn't support the pagination --- .../app/rest/repository/ExternalSourceRestRepository.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ExternalSourceRestRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ExternalSourceRestRepository.java index 49a128cd85..948e25e364 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ExternalSourceRestRepository.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ExternalSourceRestRepository.java @@ -89,10 +89,10 @@ public class ExternalSourceRestRepository extends DSpaceRestRepository findAll(Context context, Pageable pageable) { List externalSources = externalDataService.getExternalDataProviders(); - return converter.toRestPage(externalSources, pageable, externalSources.size(), - utils.obtainProjection()); + return converter.toRestPage(externalSources, pageable, utils.obtainProjection()); } public Class getDomainClass() { From 808c4633f159246543b79c723f3617e06c70c0db Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Wed, 1 Jul 2020 10:36:43 +0200 Subject: [PATCH 17/96] submit external suorce partial implementation --- .../metadatamapping/ArXivFieldMapping.java | 23 ++ .../transform/GenerateArXivQueryService.java | 53 +++ .../ArXivImportMetadataSourceServiceImpl.java | 310 ++++++++++++++++++ .../AbstractImportMetadataSourceService.java | 2 - .../spring-dspace-addon-import-services.xml | 21 +- .../config/spring/api/arxiv-integration.xml | 136 ++++++++ .../config/spring/api/external-services.xml | 9 +- 7 files changed, 547 insertions(+), 7 deletions(-) create mode 100644 dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/ArXivFieldMapping.java create mode 100644 dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/transform/GenerateArXivQueryService.java create mode 100644 dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java create mode 100644 dspace/config/spring/api/arxiv-integration.xml diff --git a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/ArXivFieldMapping.java b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/ArXivFieldMapping.java new file mode 100644 index 0000000000..c4f6996a27 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/ArXivFieldMapping.java @@ -0,0 +1,23 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.importer.external.arxiv.metadatamapping; + +import java.util.Map; +import javax.annotation.Resource; + +import org.dspace.importer.external.metadatamapping.AbstractMetadataFieldMapping; + +public class ArXivFieldMapping extends AbstractMetadataFieldMapping { + + @Override + @Resource(name = "arxivMetadataFieldMap") + public void setMetadataFieldMap(Map metadataFieldMap) { + super.setMetadataFieldMap(metadataFieldMap); + } + +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/transform/GenerateArXivQueryService.java b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/transform/GenerateArXivQueryService.java new file mode 100644 index 0000000000..7f5e08cb5a --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/transform/GenerateArXivQueryService.java @@ -0,0 +1,53 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.importer.external.arxiv.metadatamapping.transform; + +import java.util.List; + +import org.dspace.content.Item; +import org.dspace.content.MetadataValue; +import org.dspace.content.factory.ContentServiceFactory; +import org.dspace.content.service.ItemService; +import org.dspace.importer.external.datamodel.Query; +import org.dspace.importer.external.exception.MetadataSourceException; +import org.dspace.importer.external.metadatamapping.transform.GenerateQueryService; + +public class GenerateArXivQueryService implements GenerateQueryService { + + /** + * Create a Query object based on a given item. + * If the item has at least 1 value for dc.identifier.doi, the first one will be used. + * If no DOI is found, the title will be used. + * When no DOI or title is found, an null object is returned instead. + * + * @param item the Item to create a Query from + */ + @Override + public Query generateQueryForItem(Item item) throws MetadataSourceException { + Query query = new Query(); + + // Retrieve an instance of the ItemService to access business calls on an item. + ItemService itemService = ContentServiceFactory.getInstance().getItemService(); + List doi = itemService.getMetadata(item, "dc", "identifier", "doi", Item.ANY); + + if (doi.size() > 0) { + query.addParameter("term", doi.get(0).getValue()); + query.addParameter("field", "ELocationID"); + return query; + } + + List title = itemService.getMetadata(item, "dc", "title", null, Item.ANY); + + if (title.size() > 0) { + query.addParameter("term", title.get(0).getValue()); + query.addParameter("field", "title"); + return query; + } + return null; + } +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java new file mode 100644 index 0000000000..567cce1b9a --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java @@ -0,0 +1,310 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.importer.external.arxiv.service; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.io.StringReader; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.Callable; + +import org.apache.axiom.om.OMElement; +import org.apache.axiom.om.OMXMLBuilderFactory; +import org.apache.axiom.om.OMXMLParserWrapper; +import org.apache.axiom.om.xpath.AXIOMXPath; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpException; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.StatusLine; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.params.CoreConnectionPNames; +import org.apache.http.params.HttpParams; +import org.dspace.content.Item; +import org.dspace.importer.external.datamodel.ImportRecord; +import org.dspace.importer.external.datamodel.Query; +import org.dspace.importer.external.exception.MetadataSourceException; +import org.dspace.importer.external.service.AbstractImportMetadataSourceService; +import org.jaxen.JaxenException; + +public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadataSourceService { + private int timeout = 1000; + + /** + * How long to wait for a connection to be established. + * + * @param timeout milliseconds + */ + public void setTimeout(int timeout) { + this.timeout = timeout; + } + + + @Override + public Collection getRecords(String query, int start, int count) throws MetadataSourceException { + return retry(new SearchByQueryCallable(query, count, start)); + } + + @Override + public Collection getRecords(Query query) throws MetadataSourceException { + return retry(new SearchByQueryCallable(query)); + } + + @Override + public int getNbRecords(String query) throws MetadataSourceException { + List records = retry(new SearchByQueryCallable(query, null, null)); + return records != null ? records.size() : 0; + } + + @Override + public int getNbRecords(Query query) throws MetadataSourceException { + List records = retry(new SearchByQueryCallable(query)); + return records != null ? records.size() : 0; + } + + + @Override + public ImportRecord getRecord(String id) throws MetadataSourceException { + List records = retry(new SearchByIdCallable(id)); + if (records != null && records.size() > 1) { + throw new MetadataSourceException("More than one result found"); + } + return records == null ? null : records.get(0); + } + + @Override + public ImportRecord getRecord(Query query) throws MetadataSourceException { + List records = retry(new SearchByIdCallable(query)); + if (records != null && records.size() > 1) { + throw new MetadataSourceException("More than one result found"); + } + return records == null ? null : records.get(0); + } + + + @Override + public void init() throws Exception { + + } + + + + + + + @Override + public String getImportSource() { + return "arxiv"; + } + + @Override + public Collection findMatchingRecords(Item item) throws MetadataSourceException { + throw new RuntimeException(); + } + + @Override + public Collection findMatchingRecords(Query query) throws MetadataSourceException { + return null; + } + + private class SearchByQueryCallable implements Callable> { + private Query query; + + + private SearchByQueryCallable(String queryString, Integer maxResult, Integer start) { + query = new Query(); + query.addParameter("query", queryString); + query.addParameter("start", start); + query.addParameter("count", maxResult); + } + + private SearchByQueryCallable(Query query) { + this.query = query; + } + + + @Override + public List call() throws Exception { + List results = new ArrayList(); + String queryString = query.getParameterAsClass("query", String.class); + Integer start = query.getParameterAsClass("start", Integer.class); + Integer maxResult = query.getParameterAsClass("count", Integer.class); + + HttpGet method = null; + try { + HttpClient client = new DefaultHttpClient(); + HttpParams params = client.getParams(); + params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, timeout); + + try { + URIBuilder uriBuilder = new URIBuilder("http://export.arxiv.org/api/query"); + uriBuilder.addParameter("search_query", queryString); + if (maxResult != null) { + uriBuilder.addParameter("max_results", String.valueOf(maxResult)); + } + if (start != null) { + uriBuilder.addParameter("start", String.valueOf(start)); + } + method = new HttpGet(uriBuilder.build()); + } catch (URISyntaxException ex) { + throw new HttpException(ex.getMessage()); + } + + // Execute the method. + HttpResponse response = client.execute(method); + StatusLine responseStatus = response.getStatusLine(); + int statusCode = responseStatus.getStatusCode(); + + if (statusCode != HttpStatus.SC_OK) { + if (statusCode == HttpStatus.SC_BAD_REQUEST) { + throw new RuntimeException("arXiv query is not valid"); + } else { + throw new RuntimeException("Http call failed: " + + responseStatus); + } + } + + try { + InputStreamReader isReader = new InputStreamReader(response.getEntity().getContent()); + BufferedReader reader = new BufferedReader(isReader); + StringBuilder sb = new StringBuilder(); + String str; + while ((str = reader.readLine()) != null) { + sb.append(str); + } + System.out.println("XML: " + sb.toString()); + List omElements = splitToRecords(sb.toString()); + for (OMElement record : omElements) { + results.add(transformSourceRecords(record)); + } + } catch (Exception e) { + throw new RuntimeException( + "ArXiv identifier is not valid or not exist"); + } + } finally { + if (method != null) { + method.releaseConnection(); + } + } + return results; + } + } + + private class SearchByIdCallable implements Callable> { + private Query query; + + private SearchByIdCallable(Query query) { + this.query = query; + } + + private SearchByIdCallable(String id) { + this.query = new Query(); + query.addParameter("id", id); + } + + @Override + public List call() throws Exception { + List results = new ArrayList(); + String arxivid = query.getParameterAsClass("id", String.class); + HttpGet method = null; + try { + HttpClient client = new DefaultHttpClient(); + HttpParams params = client.getParams(); + params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, timeout); + try { + URIBuilder uriBuilder = new URIBuilder("http://export.arxiv.org/api/query"); + if (StringUtils.isNotBlank(arxivid)) { + arxivid = arxivid.trim(); + if (arxivid.startsWith("http://arxiv.org/abs/")) { + arxivid = arxivid.substring("http://arxiv.org/abs/".length()); + } else if (arxivid.toLowerCase().startsWith("arxiv:")) { + arxivid = arxivid.substring("arxiv:".length()); + } + uriBuilder.addParameter("id_list", arxivid); + method = new HttpGet(uriBuilder.build()); + } + } catch (URISyntaxException ex) { + throw new HttpException(ex.getMessage()); + } + + // Execute the method. + HttpResponse response = client.execute(method); + StatusLine responseStatus = response.getStatusLine(); + int statusCode = responseStatus.getStatusCode(); + if (statusCode != HttpStatus.SC_OK) { + if (statusCode == HttpStatus.SC_BAD_REQUEST) { + throw new RuntimeException("arXiv query is not valid"); + } else { + throw new RuntimeException("Http call failed: " + + responseStatus); + } + } + try { + InputStreamReader isReader = new InputStreamReader(response.getEntity().getContent()); + BufferedReader reader = new BufferedReader(isReader); + StringBuffer sb = new StringBuffer(); + String str; + while ((str = reader.readLine()) != null) { + sb.append(str); + } + List omElements = splitToRecords(sb.toString()); + for (OMElement record : omElements) { + results.add(transformSourceRecords(record)); + } + } catch (Exception e) { + throw new RuntimeException( + "ArXiv identifier is not valid or not exist"); + } + } finally { + if (method != null) { + method.releaseConnection(); + } + } + return results; + } + } + + private class FindMatchingRecordCallable implements Callable> { + private Query query; + + private FindMatchingRecordCallable(Item item) throws MetadataSourceException { + query = getGenerateQueryForItem().generateQueryForItem(item); + } + + public FindMatchingRecordCallable(Query q) { + query = q; + } + + @Override + public List call() throws Exception { + return null; + } + } + + private static List splitToRecords(String recordsSrc) { + OMXMLParserWrapper records = OMXMLBuilderFactory.createOMBuilder(new StringReader(recordsSrc)); + OMElement element = records.getDocumentElement(); + AXIOMXPath xpath = null; + try { + xpath = new AXIOMXPath("ns:entry"); + xpath.addNamespace("ns", "http://www.w3.org/2005/Atom"); + List recordsList = xpath.selectNodes(element); + return recordsList; + } catch (JaxenException e) { + return null; + } + } + + +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/AbstractImportMetadataSourceService.java b/dspace-api/src/main/java/org/dspace/importer/external/service/AbstractImportMetadataSourceService.java index a803958a9d..3bf76438cd 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/service/AbstractImportMetadataSourceService.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/AbstractImportMetadataSourceService.java @@ -16,7 +16,6 @@ import org.dspace.importer.external.metadatamapping.contributor.MetadataContribu import org.dspace.importer.external.metadatamapping.transform.GenerateQueryService; import org.dspace.importer.external.service.components.AbstractRemoteMetadataSource; import org.dspace.importer.external.service.components.MetadataSource; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Required; /** @@ -49,7 +48,6 @@ public abstract class AbstractImportMetadataSourceService extends Ab * * @param generateQueryForItem the query generator to be used. */ - @Autowired public void setGenerateQueryForItem(GenerateQueryService generateQueryForItem) { this.generateQueryForItem = generateQueryForItem; } diff --git a/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml b/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml index bbdf085619..a351280b98 100644 --- a/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml +++ b/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml @@ -19,10 +19,6 @@ - - - + + + + + + + + + + + + + + + + + + + + Defines which metadatum is mapped on which metadatum. Note that while the key must be unique it + only matters here for postprocessing of the value. The mapped MetadatumContributor has full control over + what metadatafield is generated. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Defines how an org.dspace.content.Item is mapped to a query in scopus. Please note that exactly one of + these must be present. If multiple are present the result is undefined. + + + + \ No newline at end of file diff --git a/dspace/config/spring/api/external-services.xml b/dspace/config/spring/api/external-services.xml index 098b53c2ca..af24e41980 100644 --- a/dspace/config/spring/api/external-services.xml +++ b/dspace/config/spring/api/external-services.xml @@ -31,10 +31,17 @@ - + + + + + + + + From 238317bc8df250130af162de845623702b43cd81 Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Wed, 1 Jul 2020 16:50:45 +0200 Subject: [PATCH 18/96] Arxiv Live import integration --- .../ArXivImportMetadataSourceServiceImpl.java | 207 ++++++------------ .../importer/external/datamodel/Query.java | 2 +- .../config/spring/api/arxiv-integration.xml | 49 +++-- 3 files changed, 96 insertions(+), 162 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java index 567cce1b9a..3ef9a2999f 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java @@ -7,30 +7,23 @@ */ package org.dspace.importer.external.arxiv.service; -import java.io.BufferedReader; -import java.io.InputStreamReader; import java.io.StringReader; -import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.concurrent.Callable; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.Invocation; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; import org.apache.axiom.om.OMElement; import org.apache.axiom.om.OMXMLBuilderFactory; import org.apache.axiom.om.OMXMLParserWrapper; import org.apache.axiom.om.xpath.AXIOMXPath; import org.apache.commons.lang3.StringUtils; -import org.apache.http.HttpException; -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.StatusLine; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.params.CoreConnectionPNames; -import org.apache.http.params.HttpParams; import org.dspace.content.Item; import org.dspace.importer.external.datamodel.ImportRecord; import org.dspace.importer.external.datamodel.Query; @@ -39,17 +32,8 @@ import org.dspace.importer.external.service.AbstractImportMetadataSourceService; import org.jaxen.JaxenException; public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadataSourceService { - private int timeout = 1000; - - /** - * How long to wait for a connection to be established. - * - * @param timeout milliseconds - */ - public void setTimeout(int timeout) { - this.timeout = timeout; - } + private WebTarget webTarget; @Override public Collection getRecords(String query, int start, int count) throws MetadataSourceException { @@ -95,7 +79,8 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata @Override public void init() throws Exception { - + Client client = ClientBuilder.newClient(); + webTarget = client.target("http://export.arxiv.org/api/query"); } @@ -115,7 +100,7 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata @Override public Collection findMatchingRecords(Query query) throws MetadataSourceException { - return null; + return retry(new FindMatchingRecordCallable(query)); } private class SearchByQueryCallable implements Callable> { @@ -140,62 +125,19 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata String queryString = query.getParameterAsClass("query", String.class); Integer start = query.getParameterAsClass("start", Integer.class); Integer maxResult = query.getParameterAsClass("count", Integer.class); - - HttpGet method = null; - try { - HttpClient client = new DefaultHttpClient(); - HttpParams params = client.getParams(); - params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, timeout); - - try { - URIBuilder uriBuilder = new URIBuilder("http://export.arxiv.org/api/query"); - uriBuilder.addParameter("search_query", queryString); - if (maxResult != null) { - uriBuilder.addParameter("max_results", String.valueOf(maxResult)); - } - if (start != null) { - uriBuilder.addParameter("start", String.valueOf(start)); - } - method = new HttpGet(uriBuilder.build()); - } catch (URISyntaxException ex) { - throw new HttpException(ex.getMessage()); - } - - // Execute the method. - HttpResponse response = client.execute(method); - StatusLine responseStatus = response.getStatusLine(); - int statusCode = responseStatus.getStatusCode(); - - if (statusCode != HttpStatus.SC_OK) { - if (statusCode == HttpStatus.SC_BAD_REQUEST) { - throw new RuntimeException("arXiv query is not valid"); - } else { - throw new RuntimeException("Http call failed: " - + responseStatus); - } - } - - try { - InputStreamReader isReader = new InputStreamReader(response.getEntity().getContent()); - BufferedReader reader = new BufferedReader(isReader); - StringBuilder sb = new StringBuilder(); - String str; - while ((str = reader.readLine()) != null) { - sb.append(str); - } - System.out.println("XML: " + sb.toString()); - List omElements = splitToRecords(sb.toString()); - for (OMElement record : omElements) { - results.add(transformSourceRecords(record)); - } - } catch (Exception e) { - throw new RuntimeException( - "ArXiv identifier is not valid or not exist"); - } - } finally { - if (method != null) { - method.releaseConnection(); - } + WebTarget local = webTarget.queryParam("search_query", queryString); + if (maxResult != null) { + local = local.queryParam("max_results", String.valueOf(maxResult)); + } + if (start != null) { + local = local.queryParam("start", String.valueOf(start)); + } + Invocation.Builder invocationBuilder = local.request(MediaType.TEXT_PLAIN_TYPE); + Response response = invocationBuilder.get(); + String responseString = response.readEntity(String.class); + List omElements = splitToRecords(responseString); + for (OMElement record : omElements) { + results.add(transformSourceRecords(record)); } return results; } @@ -217,78 +159,64 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata public List call() throws Exception { List results = new ArrayList(); String arxivid = query.getParameterAsClass("id", String.class); - HttpGet method = null; - try { - HttpClient client = new DefaultHttpClient(); - HttpParams params = client.getParams(); - params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, timeout); - try { - URIBuilder uriBuilder = new URIBuilder("http://export.arxiv.org/api/query"); - if (StringUtils.isNotBlank(arxivid)) { - arxivid = arxivid.trim(); - if (arxivid.startsWith("http://arxiv.org/abs/")) { - arxivid = arxivid.substring("http://arxiv.org/abs/".length()); - } else if (arxivid.toLowerCase().startsWith("arxiv:")) { - arxivid = arxivid.substring("arxiv:".length()); - } - uriBuilder.addParameter("id_list", arxivid); - method = new HttpGet(uriBuilder.build()); - } - } catch (URISyntaxException ex) { - throw new HttpException(ex.getMessage()); - } - - // Execute the method. - HttpResponse response = client.execute(method); - StatusLine responseStatus = response.getStatusLine(); - int statusCode = responseStatus.getStatusCode(); - if (statusCode != HttpStatus.SC_OK) { - if (statusCode == HttpStatus.SC_BAD_REQUEST) { - throw new RuntimeException("arXiv query is not valid"); - } else { - throw new RuntimeException("Http call failed: " - + responseStatus); - } - } - try { - InputStreamReader isReader = new InputStreamReader(response.getEntity().getContent()); - BufferedReader reader = new BufferedReader(isReader); - StringBuffer sb = new StringBuffer(); - String str; - while ((str = reader.readLine()) != null) { - sb.append(str); - } - List omElements = splitToRecords(sb.toString()); - for (OMElement record : omElements) { - results.add(transformSourceRecords(record)); - } - } catch (Exception e) { - throw new RuntimeException( - "ArXiv identifier is not valid or not exist"); - } - } finally { - if (method != null) { - method.releaseConnection(); + if (StringUtils.isNotBlank(arxivid)) { + arxivid = arxivid.trim(); + if (arxivid.startsWith("http://arxiv.org/abs/")) { + arxivid = arxivid.substring("http://arxiv.org/abs/".length()); + } else if (arxivid.toLowerCase().startsWith("arxiv:")) { + arxivid = arxivid.substring("arxiv:".length()); } } + WebTarget local = webTarget.queryParam("id_list", arxivid); + Invocation.Builder invocationBuilder = local.request(MediaType.TEXT_PLAIN_TYPE); + Response response = invocationBuilder.get(); + String responseString = response.readEntity(String.class); + List omElements = splitToRecords(responseString); + for (OMElement record : omElements) { + results.add(transformSourceRecords(record)); + } return results; } } private class FindMatchingRecordCallable implements Callable> { + private Query query; - private FindMatchingRecordCallable(Item item) throws MetadataSourceException { - query = getGenerateQueryForItem().generateQueryForItem(item); - } - - public FindMatchingRecordCallable(Query q) { + private FindMatchingRecordCallable(Query q) { query = q; } @Override public List call() throws Exception { - return null; + String queryString = getQuery(this.query); + List results = new ArrayList(); + WebTarget local = webTarget.queryParam("search_query", queryString); + Invocation.Builder invocationBuilder = local.request(MediaType.TEXT_PLAIN_TYPE); + Response response = invocationBuilder.get(); + String responseString = response.readEntity(String.class); + List omElements = splitToRecords(responseString); + for (OMElement record : omElements) { + results.add(transformSourceRecords(record)); + } + return results; + } + + private String getQuery(Query query) { + String title = query.getParameterAsClass("title", String.class); + String author = query.getParameterAsClass("author", String.class); + StringBuffer queryString = new StringBuffer(); + if (StringUtils.isNotBlank(title)) { + queryString.append("ti:\"").append(title).append("\""); + } + if (StringUtils.isNotBlank(author)) { + // [FAU] + if (queryString.length() > 0) { + queryString.append(" AND "); + } + queryString.append("au:\"").append(author).append("\""); + } + return queryString.toString(); } } @@ -306,5 +234,4 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata } } - } diff --git a/dspace-api/src/main/java/org/dspace/importer/external/datamodel/Query.java b/dspace-api/src/main/java/org/dspace/importer/external/datamodel/Query.java index 8c5e1b394a..8f392bdb52 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/datamodel/Query.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/datamodel/Query.java @@ -71,7 +71,7 @@ public class Query { return null; } else { Object o = c.iterator().next(); - if (clazz.isAssignableFrom(o.getClass())) { + if (o != null && clazz.isAssignableFrom(o.getClass())) { return (T) o; } else { return null; diff --git a/dspace/config/spring/api/arxiv-integration.xml b/dspace/config/spring/api/arxiv-integration.xml index d16a1ae52a..a22bfe9eeb 100644 --- a/dspace/config/spring/api/arxiv-integration.xml +++ b/dspace/config/spring/api/arxiv-integration.xml @@ -30,55 +30,55 @@ - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + @@ -118,8 +118,15 @@ - - + + + + + + + @@ -133,4 +140,4 @@ - \ No newline at end of file + From b15fa7c5b0c381dd05d6493d0b5dd3e08a402c02 Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Thu, 2 Jul 2020 02:32:11 +0200 Subject: [PATCH 19/96] show ids and links in response --- .../ArXivIdMetadataContributor.java | 180 ++++++++++++++++++ .../ArXivImportMetadataSourceServiceImpl.java | 165 ++++++++++++++-- .../spring-dspace-addon-import-services.xml | 2 +- .../config/spring/api/arxiv-integration.xml | 131 ++++++------- 4 files changed, 384 insertions(+), 94 deletions(-) create mode 100644 dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/contributor/ArXivIdMetadataContributor.java diff --git a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/contributor/ArXivIdMetadataContributor.java b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/contributor/ArXivIdMetadataContributor.java new file mode 100644 index 0000000000..018cde8ac7 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/contributor/ArXivIdMetadataContributor.java @@ -0,0 +1,180 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.importer.external.arxiv.metadatamapping.contributor; + +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import javax.annotation.Resource; + +import org.apache.axiom.om.OMAttribute; +import org.apache.axiom.om.OMElement; +import org.apache.axiom.om.OMText; +import org.apache.axiom.om.xpath.AXIOMXPath; +import org.dspace.importer.external.metadatamapping.MetadataFieldConfig; +import org.dspace.importer.external.metadatamapping.MetadataFieldMapping; +import org.dspace.importer.external.metadatamapping.MetadatumDTO; +import org.dspace.importer.external.metadatamapping.contributor.MetadataContributor; +import org.jaxen.JaxenException; +import org.springframework.beans.factory.annotation.Required; + +public class ArXivIdMetadataContributor implements MetadataContributor { + private MetadataFieldConfig field; + + /** + * Return prefixToNamespaceMapping + * + * @return a prefixToNamespaceMapping map + */ + public Map getPrefixToNamespaceMapping() { + return prefixToNamespaceMapping; + } + + private MetadataFieldMapping> metadataFieldMapping; + + /** + * Return metadataFieldMapping + * + * @return MetadataFieldMapping + */ + public MetadataFieldMapping> getMetadataFieldMapping() { + return metadataFieldMapping; + } + + /** + * Set the metadataFieldMapping of this ArXivIdMetadataContributor + * + * @param metadataFieldMapping the new mapping. + */ + public void setMetadataFieldMapping( + MetadataFieldMapping> metadataFieldMapping) { + this.metadataFieldMapping = metadataFieldMapping; + } + + /** + * Set the prefixToNamespaceMapping for this object, + * + * @param prefixToNamespaceMapping the new mapping. + */ + @Resource(name = "isiFullprefixMapping") + public void setPrefixToNamespaceMapping(Map prefixToNamespaceMapping) { + this.prefixToNamespaceMapping = prefixToNamespaceMapping; + } + + private Map prefixToNamespaceMapping; + + /** + * Initialize ArXivIdMetadataContributor with a query, prefixToNamespaceMapping and MetadataFieldConfig + * + * @param query query string + * @param prefixToNamespaceMapping metadata prefix to namespace mapping + * @param field + * MetadataFieldConfig + */ + public ArXivIdMetadataContributor(String query, Map prefixToNamespaceMapping, + MetadataFieldConfig field) { + this.query = query; + this.prefixToNamespaceMapping = prefixToNamespaceMapping; + this.field = field; + } + + /** + * Empty constructor for ArXivIdMetadataContributor + */ + public ArXivIdMetadataContributor() { + + } + + private String query; + + /** + * Return the MetadataFieldConfig used while retrieving MetadatumDTO + * + * @return MetadataFieldConfig + */ + public MetadataFieldConfig getField() { + return field; + } + + /** + * Setting the MetadataFieldConfig + * + * @param field MetadataFieldConfig used while retrieving MetadatumDTO + */ + @Required + public void setField(MetadataFieldConfig field) { + this.field = field; + } + + /** + * Return query used to create an xpathExpression on, this query is used to + * + * @return the query this instance is based on + */ + public String getQuery() { + return query; + } + + @Required + public void setQuery(String query) { + this.query = query; + } + + /** + * Retrieve the metadata associated with the given object. + * Depending on the retrieved node (using the query), different types of values will be added to the MetadatumDTO + * list + * + * @param t A class to retrieve metadata from. + * @return a collection of import records. Only the identifier of the found records may be put in the record. + */ + @Override + public Collection contributeMetadata(OMElement t) { + List values = new LinkedList<>(); + try { + AXIOMXPath xpath = new AXIOMXPath(query); + for (String ns : prefixToNamespaceMapping.keySet()) { + xpath.addNamespace(prefixToNamespaceMapping.get(ns), ns); + } + List nodes = xpath.selectNodes(t); + for (Object el : nodes) { + if (el instanceof OMElement) { + values.add(metadataFieldMapping.toDCValue(field, ((OMElement) el).getText())); + } else if (el instanceof OMAttribute) { + values.add(metadataFieldMapping.toDCValue(field, ((OMAttribute) el).getAttributeValue())); + } else if (el instanceof String) { + values.add(metadataFieldMapping.toDCValue(field, (String) el)); + } else if (el instanceof OMText) { + values.add(metadataFieldMapping.toDCValue(field, ((OMText) el).getText())); + } else { + System.err.println("node of type: " + el.getClass()); + } + } + parseValue(values); + return values; + } catch (JaxenException e) { + System.err.println(query); + throw new RuntimeException(e); + } + } + + private void parseValue(List dtos) { + if (dtos != null) { + for (MetadatumDTO dto : dtos) { + if (dto != null && dto.getValue() != null) { + int startIndex = dto.getValue().lastIndexOf('/'); + int endIndex = dto.getValue().length(); + String id = dto.getValue().substring(startIndex + 1, endIndex); + dto.setValue(id); + } + } + } + } + +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java index 3ef9a2999f..429a54a013 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java @@ -31,78 +31,185 @@ import org.dspace.importer.external.exception.MetadataSourceException; import org.dspace.importer.external.service.AbstractImportMetadataSourceService; import org.jaxen.JaxenException; +/** + * Implements a data source for querying ArXiv + * + * @author Pasquale Cavallo (pasquale.cavallo at 4Science dot it) + * + */ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadataSourceService { private WebTarget webTarget; + private String baseAddress; + /** + * Find the number of records matching a string query. Supports pagination + * + * @param query a query string to base the search on. + * @param start offset to start at + * @param count number of records to retrieve. + * @return a set of records. Fully transformed. + * @throws MetadataSourceException if the underlying methods throw any exception. + */ @Override public Collection getRecords(String query, int start, int count) throws MetadataSourceException { return retry(new SearchByQueryCallable(query, count, start)); } + /** + * Find records based on a object query. + * + * @param query a query object to base the search on. + * @return a set of records. Fully transformed. + * @throws MetadataSourceException if the underlying methods throw any exception. + */ @Override public Collection getRecords(Query query) throws MetadataSourceException { return retry(new SearchByQueryCallable(query)); } + /** + * Find the number of records matching a query; + * + * @param query a query object to base the search on. + * @return the sum of the matching records over this import source + * @throws MetadataSourceException if the underlying methods throw any exception. + */ @Override public int getNbRecords(String query) throws MetadataSourceException { - List records = retry(new SearchByQueryCallable(query, null, null)); - return records != null ? records.size() : 0; + return retry(new CountByQueryCallable(query)); } + + /** + * Find the number of records matching a query; + * + * @param query a query string to base the search on. + * @return the sum of the matching records over this import source + * @throws MetadataSourceException if the underlying methods throw any exception. + */ @Override public int getNbRecords(Query query) throws MetadataSourceException { - List records = retry(new SearchByQueryCallable(query)); - return records != null ? records.size() : 0; + return retry(new CountByQueryCallable(query)); } + /** + * Get a single record from the source by id + * + * @param id id of the record in ArXiv + * @return the first matching record + * @throws MetadataSourceException if the underlying methods throw any exception. + */ @Override public ImportRecord getRecord(String id) throws MetadataSourceException { List records = retry(new SearchByIdCallable(id)); - if (records != null && records.size() > 1) { - throw new MetadataSourceException("More than one result found"); - } - return records == null ? null : records.get(0); + return records == null || records.isEmpty() ? null : records.get(0); } + /** + * Get a single record from the source. + * + * @param query a query matching a single record + * @return the first matching record + * @throws MetadataSourceException if the underlying methods throw any exception. + */ @Override public ImportRecord getRecord(Query query) throws MetadataSourceException { List records = retry(new SearchByIdCallable(query)); - if (records != null && records.size() > 1) { - throw new MetadataSourceException("More than one result found"); - } - return records == null ? null : records.get(0); + return records == null || records.isEmpty() ? null : records.get(0); } - + /** + * Initialize the class + * + * @throws Exception on generic exception + */ @Override public void init() throws Exception { Client client = ClientBuilder.newClient(); - webTarget = client.target("http://export.arxiv.org/api/query"); + webTarget = client.target(baseAddress); } - - - - - + /** + * The string that identifies this import implementation. Preferable a URI + * + * @return the identifying uri + */ @Override public String getImportSource() { return "arxiv"; } + /** + * NOT IMPLEMENTED: Finds records based on an item + * + * @param item an item to base the search on + * @return a collection of import records. Only the identifier of the found records may be put in the record. + * @throws MetadataSourceException if the underlying methods throw any exception. + */ @Override public Collection findMatchingRecords(Item item) throws MetadataSourceException { throw new RuntimeException(); } + /** + * Finds records based on query object. + * Supports search by title and/or author + * + * @param query a query object to base the search on. + * @return a collection of import records. + * @throws MetadataSourceException if the underlying methods throw any exception. + */ @Override public Collection findMatchingRecords(Query query) throws MetadataSourceException { return retry(new FindMatchingRecordCallable(query)); } + private class CountByQueryCallable implements Callable { + private Query query; + + + private CountByQueryCallable(String queryString) { + query = new Query(); + query.addParameter("query", queryString); + } + + private CountByQueryCallable(Query query) { + this.query = query; + } + + + @Override + public Integer call() throws Exception { + String queryString = query.getParameterAsClass("query", String.class); + Integer start = query.getParameterAsClass("start", Integer.class); + Integer maxResult = query.getParameterAsClass("count", Integer.class); + WebTarget local = webTarget.queryParam("search_query", queryString); + if (maxResult != null) { + local = local.queryParam("max_results", String.valueOf(maxResult)); + } + if (start != null) { + local = local.queryParam("start", String.valueOf(start)); + } + Invocation.Builder invocationBuilder = local.request(MediaType.TEXT_PLAIN_TYPE); + Response response = invocationBuilder.get(); + String responseString = response.readEntity(String.class); + OMXMLParserWrapper records = OMXMLBuilderFactory.createOMBuilder(new StringReader(responseString)); + OMElement element = records.getDocumentElement(); + AXIOMXPath xpath = null; + try { + xpath = new AXIOMXPath("opensearch:totalResults"); + xpath.addNamespace("opensearch", "http://a9.com/-/spec/opensearch/1.1/"); + OMElement count = (OMElement) xpath.selectSingleNode(element); + return Integer.parseInt(count.getText()); + } catch (JaxenException e) { + return null; + } + } + } + + private class SearchByQueryCallable implements Callable> { private Query query; @@ -220,7 +327,7 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata } } - private static List splitToRecords(String recordsSrc) { + private List splitToRecords(String recordsSrc) { OMXMLParserWrapper records = OMXMLBuilderFactory.createOMBuilder(new StringReader(recordsSrc)); OMElement element = records.getDocumentElement(); AXIOMXPath xpath = null; @@ -234,4 +341,22 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata } } + /** + * Return the baseAddress set to this object + * + * @return The String object that represents the baseAddress of this object + */ + public String getBaseAddress() { + return baseAddress; + } + + /** + * Set the baseAddress to this object + * + * @param baseAddress The String object that represents the baseAddress of this object + */ + public void setBaseAddress(String baseAddress) { + this.baseAddress = baseAddress; + } + } diff --git a/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml b/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml index a351280b98..d2d6dd10c1 100644 --- a/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml +++ b/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml @@ -37,7 +37,7 @@ class="org.dspace.importer.external.arxiv.service.ArXivImportMetadataSourceServiceImpl" scope="singleton"> - + diff --git a/dspace/config/spring/api/arxiv-integration.xml b/dspace/config/spring/api/arxiv-integration.xml index a22bfe9eeb..9b453e3fc9 100644 --- a/dspace/config/spring/api/arxiv-integration.xml +++ b/dspace/config/spring/api/arxiv-integration.xml @@ -17,106 +17,92 @@ only matters here for postprocessing of the value. The mapped MetadatumContributor has full control over what metadatafield is generated. - - - - - - + - - + + + + + + - - - - - - - - - - - - + + - - - - - + + - - - - - - - + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -129,7 +115,6 @@ - From f71866b5e300453aec669e24ea087204d93ff90c Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Thu, 2 Jul 2020 10:30:49 +0200 Subject: [PATCH 20/96] Add comment --- .../contributor/ArXivIdMetadataContributor.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/contributor/ArXivIdMetadataContributor.java b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/contributor/ArXivIdMetadataContributor.java index 018cde8ac7..ed38f955c8 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/contributor/ArXivIdMetadataContributor.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/contributor/ArXivIdMetadataContributor.java @@ -24,6 +24,13 @@ import org.dspace.importer.external.metadatamapping.contributor.MetadataContribu import org.jaxen.JaxenException; import org.springframework.beans.factory.annotation.Required; +/** + * Arxiv specific implementation of {@link MetadataContributor} + * Responsible for generating the ArXiv Id from the retrieved item. + * + * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it) + * + */ public class ArXivIdMetadataContributor implements MetadataContributor { private MetadataFieldConfig field; From 03ac31dc109d733917c94f1bf1cb8d26a6f14bcf Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Thu, 2 Jul 2020 12:05:50 +0200 Subject: [PATCH 21/96] Exception handling --- .../transform/GenerateArXivQueryService.java | 53 ------------------- .../ArXivImportMetadataSourceServiceImpl.java | 3 +- .../DSpaceApiExceptionControllerAdvice.java | 9 ++++ .../config/spring/api/arxiv-integration.xml | 9 ---- 4 files changed, 11 insertions(+), 63 deletions(-) delete mode 100644 dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/transform/GenerateArXivQueryService.java diff --git a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/transform/GenerateArXivQueryService.java b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/transform/GenerateArXivQueryService.java deleted file mode 100644 index 7f5e08cb5a..0000000000 --- a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/transform/GenerateArXivQueryService.java +++ /dev/null @@ -1,53 +0,0 @@ -/** - * The contents of this file are subject to the license and copyright - * detailed in the LICENSE and NOTICE files at the root of the source - * tree and available online at - * - * http://www.dspace.org/license/ - */ -package org.dspace.importer.external.arxiv.metadatamapping.transform; - -import java.util.List; - -import org.dspace.content.Item; -import org.dspace.content.MetadataValue; -import org.dspace.content.factory.ContentServiceFactory; -import org.dspace.content.service.ItemService; -import org.dspace.importer.external.datamodel.Query; -import org.dspace.importer.external.exception.MetadataSourceException; -import org.dspace.importer.external.metadatamapping.transform.GenerateQueryService; - -public class GenerateArXivQueryService implements GenerateQueryService { - - /** - * Create a Query object based on a given item. - * If the item has at least 1 value for dc.identifier.doi, the first one will be used. - * If no DOI is found, the title will be used. - * When no DOI or title is found, an null object is returned instead. - * - * @param item the Item to create a Query from - */ - @Override - public Query generateQueryForItem(Item item) throws MetadataSourceException { - Query query = new Query(); - - // Retrieve an instance of the ItemService to access business calls on an item. - ItemService itemService = ContentServiceFactory.getInstance().getItemService(); - List doi = itemService.getMetadata(item, "dc", "identifier", "doi", Item.ANY); - - if (doi.size() > 0) { - query.addParameter("term", doi.get(0).getValue()); - query.addParameter("field", "ELocationID"); - return query; - } - - List title = itemService.getMetadata(item, "dc", "title", null, Item.ANY); - - if (title.size() > 0) { - query.addParameter("term", title.get(0).getValue()); - query.addParameter("field", "title"); - return query; - } - return null; - } -} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java index 429a54a013..2995466c3c 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java @@ -27,6 +27,7 @@ import org.apache.commons.lang3.StringUtils; import org.dspace.content.Item; import org.dspace.importer.external.datamodel.ImportRecord; import org.dspace.importer.external.datamodel.Query; +import org.dspace.importer.external.exception.ExternalProviderMethodNotImplementedException; import org.dspace.importer.external.exception.MetadataSourceException; import org.dspace.importer.external.service.AbstractImportMetadataSourceService; import org.jaxen.JaxenException; @@ -150,7 +151,7 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata */ @Override public Collection findMatchingRecords(Item item) throws MetadataSourceException { - throw new RuntimeException(); + throw new ExternalProviderMethodNotImplementedException("This method is not implemented for ArXiv"); } /** diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/exception/DSpaceApiExceptionControllerAdvice.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/exception/DSpaceApiExceptionControllerAdvice.java index d255b6fe27..07cc31420e 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/exception/DSpaceApiExceptionControllerAdvice.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/exception/DSpaceApiExceptionControllerAdvice.java @@ -16,6 +16,7 @@ import javax.servlet.http.HttpServletResponse; import org.dspace.app.rest.security.RestAuthenticationService; import org.dspace.authorize.AuthorizeException; +import org.dspace.importer.external.exception.ExternalProviderMethodNotImplementedException; import org.springframework.beans.TypeMismatchException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.annotation.AnnotationUtils; @@ -38,6 +39,8 @@ import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExcep * @author Tom Desair (tom dot desair at atmire dot com) * @author Frederic Van Reet (frederic dot vanreet at atmire dot com) * @author Andrea Bollini (andrea.bollini at 4science.it) + * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it) + * */ @ControllerAdvice public class DSpaceApiExceptionControllerAdvice extends ResponseEntityExceptionHandler { @@ -45,6 +48,12 @@ public class DSpaceApiExceptionControllerAdvice extends ResponseEntityExceptionH @Autowired private RestAuthenticationService restAuthenticationService; + @ExceptionHandler(ExternalProviderMethodNotImplementedException.class) + protected void externalResourceMethodNotImplementedException(HttpServletRequest request, + HttpServletResponse response, ExternalProviderMethodNotImplementedException ex) throws IOException { + sendErrorResponse(request, response, ex, ex.getMessage(), HttpServletResponse.SC_NOT_IMPLEMENTED); + } + @ExceptionHandler({AuthorizeException.class, RESTAuthorizationException.class, AccessDeniedException.class}) protected void handleAuthorizeException(HttpServletRequest request, HttpServletResponse response, Exception ex) throws IOException { diff --git a/dspace/config/spring/api/arxiv-integration.xml b/dspace/config/spring/api/arxiv-integration.xml index 9b453e3fc9..e963e73a20 100644 --- a/dspace/config/spring/api/arxiv-integration.xml +++ b/dspace/config/spring/api/arxiv-integration.xml @@ -101,9 +101,6 @@ - - - @@ -119,10 +116,4 @@ - - Defines how an org.dspace.content.Item is mapped to a query in scopus. Please note that exactly one of - these must be present. If multiple are present the result is undefined. - - - From 02953eb087c931dba77fd84d3e6ba788464b4458 Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Thu, 2 Jul 2020 14:45:04 +0200 Subject: [PATCH 22/96] Update comment and minor fix --- .../metadatamapping/ArXivFieldMapping.java | 14 +++++++ .../ArXivIdMetadataContributor.java | 6 +-- ...ProviderMethodNotImplementedException.java | 40 +++++++++++++++++++ 3 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 dspace-api/src/main/java/org/dspace/importer/external/exception/ExternalProviderMethodNotImplementedException.java diff --git a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/ArXivFieldMapping.java b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/ArXivFieldMapping.java index c4f6996a27..272b149015 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/ArXivFieldMapping.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/ArXivFieldMapping.java @@ -12,8 +12,22 @@ import javax.annotation.Resource; import org.dspace.importer.external.metadatamapping.AbstractMetadataFieldMapping; +/** + * An implementation of {@link AbstractMetadataFieldMapping} + * Responsible for defining the mapping of the ArXiv metadatum fields on the DSpace metadatum fields + * + * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it) + */ public class ArXivFieldMapping extends AbstractMetadataFieldMapping { + /** + * Defines which metadatum is mapped on which metadatum. Note that while the key must be unique it + * only matters here for postprocessing of the value. The mapped MetadatumContributor has full control over + * what metadatafield is generated. + * + * @param metadataFieldMap The map containing the link between retrieve metadata and metadata that will be set to + * the item. + */ @Override @Resource(name = "arxivMetadataFieldMap") public void setMetadataFieldMap(Map metadataFieldMap) { diff --git a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/contributor/ArXivIdMetadataContributor.java b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/contributor/ArXivIdMetadataContributor.java index ed38f955c8..727a3fcf4e 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/contributor/ArXivIdMetadataContributor.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/contributor/ArXivIdMetadataContributor.java @@ -174,10 +174,10 @@ public class ArXivIdMetadataContributor implements MetadataContributor dtos) { if (dtos != null) { for (MetadatumDTO dto : dtos) { - if (dto != null && dto.getValue() != null) { - int startIndex = dto.getValue().lastIndexOf('/'); + if (dto != null && dto.getValue() != null && dto.getValue().contains("/")) { + int startIndex = dto.getValue().lastIndexOf('/') + 1; int endIndex = dto.getValue().length(); - String id = dto.getValue().substring(startIndex + 1, endIndex); + String id = dto.getValue().substring(startIndex, endIndex); dto.setValue(id); } } diff --git a/dspace-api/src/main/java/org/dspace/importer/external/exception/ExternalProviderMethodNotImplementedException.java b/dspace-api/src/main/java/org/dspace/importer/external/exception/ExternalProviderMethodNotImplementedException.java new file mode 100644 index 0000000000..e8df7a3861 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/exception/ExternalProviderMethodNotImplementedException.java @@ -0,0 +1,40 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.importer.external.exception; + +/** + * Exception used when an External Provider don't implements the invoked method + * + * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it) + * + */ +public class ExternalProviderMethodNotImplementedException extends RuntimeException { + + private static final long serialVersionUID = 5268699485635863003L; + + public ExternalProviderMethodNotImplementedException() { + super(); + } + + public ExternalProviderMethodNotImplementedException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } + + public ExternalProviderMethodNotImplementedException(String message, Throwable cause) { + super(message, cause); + } + + public ExternalProviderMethodNotImplementedException(String message) { + super(message); + } + + public ExternalProviderMethodNotImplementedException(Throwable cause) { + super(cause); + } +} From dbde82478b052371bfc1b3780a622797b3ebf333 Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Thu, 2 Jul 2020 16:22:36 +0200 Subject: [PATCH 23/96] remove unused code --- .../ArXivImportMetadataSourceServiceImpl.java | 11 +- ...ProviderMethodNotImplementedException.java | 40 --- .../submit/lookup/ArXivFileDataLoader.java | 146 -------- .../submit/lookup/ArXivOnlineDataLoader.java | 84 ----- .../dspace/submit/lookup/ArXivService.java | 159 --------- .../org/dspace/submit/lookup/ArxivUtils.java | 151 --------- .../submit/lookup/PubmedFileDataLoader.java | 148 -------- .../submit/lookup/PubmedOnlineDataLoader.java | 116 ------- .../dspace/submit/lookup/PubmedService.java | 265 --------------- .../org/dspace/submit/lookup/PubmedUtils.java | 316 ------------------ .../spring-dspace-addon-import-services.xml | 1 - .../DSpaceApiExceptionControllerAdvice.java | 7 - dspace/config/spring/api/bte.xml | 107 ------ .../spring/api/step-processing-listener.xml | 2 - 14 files changed, 4 insertions(+), 1549 deletions(-) delete mode 100644 dspace-api/src/main/java/org/dspace/importer/external/exception/ExternalProviderMethodNotImplementedException.java delete mode 100644 dspace-api/src/main/java/org/dspace/submit/lookup/ArXivFileDataLoader.java delete mode 100644 dspace-api/src/main/java/org/dspace/submit/lookup/ArXivOnlineDataLoader.java delete mode 100644 dspace-api/src/main/java/org/dspace/submit/lookup/ArXivService.java delete mode 100644 dspace-api/src/main/java/org/dspace/submit/lookup/ArxivUtils.java delete mode 100644 dspace-api/src/main/java/org/dspace/submit/lookup/PubmedFileDataLoader.java delete mode 100644 dspace-api/src/main/java/org/dspace/submit/lookup/PubmedOnlineDataLoader.java delete mode 100644 dspace-api/src/main/java/org/dspace/submit/lookup/PubmedService.java delete mode 100644 dspace-api/src/main/java/org/dspace/submit/lookup/PubmedUtils.java diff --git a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java index 2995466c3c..863a8144b4 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java @@ -12,6 +12,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.concurrent.Callable; +import javax.el.MethodNotFoundException; import javax.ws.rs.client.Client; import javax.ws.rs.client.ClientBuilder; import javax.ws.rs.client.Invocation; @@ -27,7 +28,6 @@ import org.apache.commons.lang3.StringUtils; import org.dspace.content.Item; import org.dspace.importer.external.datamodel.ImportRecord; import org.dspace.importer.external.datamodel.Query; -import org.dspace.importer.external.exception.ExternalProviderMethodNotImplementedException; import org.dspace.importer.external.exception.MetadataSourceException; import org.dspace.importer.external.service.AbstractImportMetadataSourceService; import org.jaxen.JaxenException; @@ -143,15 +143,12 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata } /** - * NOT IMPLEMENTED: Finds records based on an item - * - * @param item an item to base the search on - * @return a collection of import records. Only the identifier of the found records may be put in the record. - * @throws MetadataSourceException if the underlying methods throw any exception. + * Expect this method will be not used and erased from the interface soon */ @Override public Collection findMatchingRecords(Item item) throws MetadataSourceException { - throw new ExternalProviderMethodNotImplementedException("This method is not implemented for ArXiv"); + // FIXME: we need this method? + throw new MethodNotFoundException("This method is not implemented for ArXiv"); } /** diff --git a/dspace-api/src/main/java/org/dspace/importer/external/exception/ExternalProviderMethodNotImplementedException.java b/dspace-api/src/main/java/org/dspace/importer/external/exception/ExternalProviderMethodNotImplementedException.java deleted file mode 100644 index e8df7a3861..0000000000 --- a/dspace-api/src/main/java/org/dspace/importer/external/exception/ExternalProviderMethodNotImplementedException.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * The contents of this file are subject to the license and copyright - * detailed in the LICENSE and NOTICE files at the root of the source - * tree and available online at - * - * http://www.dspace.org/license/ - */ -package org.dspace.importer.external.exception; - -/** - * Exception used when an External Provider don't implements the invoked method - * - * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it) - * - */ -public class ExternalProviderMethodNotImplementedException extends RuntimeException { - - private static final long serialVersionUID = 5268699485635863003L; - - public ExternalProviderMethodNotImplementedException() { - super(); - } - - public ExternalProviderMethodNotImplementedException(String message, Throwable cause, boolean enableSuppression, - boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - } - - public ExternalProviderMethodNotImplementedException(String message, Throwable cause) { - super(message, cause); - } - - public ExternalProviderMethodNotImplementedException(String message) { - super(message); - } - - public ExternalProviderMethodNotImplementedException(Throwable cause) { - super(cause); - } -} diff --git a/dspace-api/src/main/java/org/dspace/submit/lookup/ArXivFileDataLoader.java b/dspace-api/src/main/java/org/dspace/submit/lookup/ArXivFileDataLoader.java deleted file mode 100644 index ebc898e4cf..0000000000 --- a/dspace-api/src/main/java/org/dspace/submit/lookup/ArXivFileDataLoader.java +++ /dev/null @@ -1,146 +0,0 @@ -/** - * The contents of this file are subject to the license and copyright - * detailed in the LICENSE and NOTICE files at the root of the source - * tree and available online at - * - * http://www.dspace.org/license/ - */ - -package org.dspace.submit.lookup; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; -import java.util.Map; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; - -import gr.ekt.bte.core.DataLoadingSpec; -import gr.ekt.bte.core.Record; -import gr.ekt.bte.core.RecordSet; -import gr.ekt.bte.core.Value; -import gr.ekt.bte.dataloader.FileDataLoader; -import gr.ekt.bte.exceptions.MalformedSourceException; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.Logger; -import org.dspace.app.util.XMLUtils; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.xml.sax.SAXException; - -/** - * @author Andrea Bollini - * @author Kostas Stamatis - * @author Luigi Andrea Pascarelli - * @author Panagiotis Koutsourakis - */ -public class ArXivFileDataLoader extends FileDataLoader { - - private static Logger log = org.apache.logging.log4j.LogManager.getLogger(ArXivFileDataLoader.class); - - Map fieldMap; // mapping between service fields and local - // intermediate fields - - /** - * Empty constructor - */ - public ArXivFileDataLoader() { - } - - /** - * @param filename Name of file to load ArXiv data from. - */ - public ArXivFileDataLoader(String filename) { - super(filename); - } - - /* - * {@see gr.ekt.bte.core.DataLoader#getRecords()} - * - * @throws MalformedSourceException - */ - @Override - public RecordSet getRecords() throws MalformedSourceException { - - RecordSet recordSet = new RecordSet(); - - try { - InputStream inputStream = new FileInputStream(new File(filename)); - - DocumentBuilderFactory factory = DocumentBuilderFactory - .newInstance(); - factory.setValidating(false); - factory.setIgnoringComments(true); - factory.setIgnoringElementContentWhitespace(true); - - DocumentBuilder db = factory.newDocumentBuilder(); - Document inDoc = db.parse(inputStream); - - Element xmlRoot = inDoc.getDocumentElement(); - List dataRoots = XMLUtils.getElementList(xmlRoot, "entry"); - - for (Element dataRoot : dataRoots) { - Record record = ArxivUtils.convertArxixDomToRecord(dataRoot); - if (record != null) { - recordSet.addRecord(convertFields(record)); - } - } - } catch (FileNotFoundException e) { - log.error(e.getMessage(), e); - } catch (ParserConfigurationException e) { - log.error(e.getMessage(), e); - } catch (SAXException e) { - log.error(e.getMessage(), e); - } catch (IOException e) { - log.error(e.getMessage(), e); - } - - return recordSet; - } - - /* - * (non-Javadoc) - * - * @see - * gr.ekt.bte.core.DataLoader#getRecords(gr.ekt.bte.core.DataLoadingSpec) - */ - @Override - public RecordSet getRecords(DataLoadingSpec spec) - throws MalformedSourceException { - if (spec.getOffset() > 0) { - return new RecordSet(); - } - return getRecords(); - } - - public Record convertFields(Record publication) { - for (String fieldName : fieldMap.keySet()) { - String md = null; - if (fieldMap != null) { - md = this.fieldMap.get(fieldName); - } - - if (StringUtils.isBlank(md)) { - continue; - } else { - md = md.trim(); - } - - if (publication.isMutable()) { - List values = publication.getValues(fieldName); - publication.makeMutable().removeField(fieldName); - publication.makeMutable().addField(md, values); - } - } - - return publication; - } - - public void setFieldMap(Map fieldMap) { - this.fieldMap = fieldMap; - } -} diff --git a/dspace-api/src/main/java/org/dspace/submit/lookup/ArXivOnlineDataLoader.java b/dspace-api/src/main/java/org/dspace/submit/lookup/ArXivOnlineDataLoader.java deleted file mode 100644 index e477412621..0000000000 --- a/dspace-api/src/main/java/org/dspace/submit/lookup/ArXivOnlineDataLoader.java +++ /dev/null @@ -1,84 +0,0 @@ -/** - * The contents of this file are subject to the license and copyright - * detailed in the LICENSE and NOTICE files at the root of the source - * tree and available online at - * - * http://www.dspace.org/license/ - */ -package org.dspace.submit.lookup; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import gr.ekt.bte.core.Record; -import org.apache.http.HttpException; -import org.dspace.core.Context; - -/** - * @author Andrea Bollini - * @author Kostas Stamatis - * @author Luigi Andrea Pascarelli - * @author Panagiotis Koutsourakis - */ -public class ArXivOnlineDataLoader extends NetworkSubmissionLookupDataLoader { - protected ArXivService arXivService = new ArXivService(); - - protected boolean searchProvider = true; - - public void setArXivService(ArXivService arXivService) { - this.arXivService = arXivService; - } - - @Override - public List getSupportedIdentifiers() { - return Arrays.asList(new String[] {ARXIV, DOI}); - } - - public void setSearchProvider(boolean searchProvider) { - this.searchProvider = searchProvider; - } - - @Override - public boolean isSearchProvider() { - return searchProvider; - } - - @Override - public List getByIdentifier(Context context, - Map> keys) throws HttpException, IOException { - List results = new ArrayList(); - if (keys != null) { - Set dois = keys.get(DOI); - Set arxivids = keys.get(ARXIV); - List items = new ArrayList(); - if (dois != null && dois.size() > 0) { - items.addAll(arXivService.getByDOIs(dois)); - } - if (arxivids != null && arxivids.size() > 0) { - for (String arxivid : arxivids) { - items.add(arXivService.getByArXivIDs(arxivid)); - } - } - - for (Record item : items) { - results.add(convertFields(item)); - } - } - return results; - } - - @Override - public List search(Context context, String title, String author, - int year) throws HttpException, IOException { - List results = new ArrayList(); - List items = arXivService.searchByTerm(title, author, year); - for (Record item : items) { - results.add(convertFields(item)); - } - return results; - } -} diff --git a/dspace-api/src/main/java/org/dspace/submit/lookup/ArXivService.java b/dspace-api/src/main/java/org/dspace/submit/lookup/ArXivService.java deleted file mode 100644 index 0a32871758..0000000000 --- a/dspace-api/src/main/java/org/dspace/submit/lookup/ArXivService.java +++ /dev/null @@ -1,159 +0,0 @@ -/** - * The contents of this file are subject to the license and copyright - * detailed in the LICENSE and NOTICE files at the root of the source - * tree and available online at - * - * http://www.dspace.org/license/ - */ -package org.dspace.submit.lookup; - -import java.io.IOException; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; - -import gr.ekt.bte.core.Record; -import org.apache.commons.lang3.StringUtils; -import org.apache.http.HttpException; -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.StatusLine; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.params.CoreConnectionPNames; -import org.apache.http.params.HttpParams; -import org.dspace.app.util.XMLUtils; -import org.w3c.dom.Document; -import org.w3c.dom.Element; - -/** - * @author Andrea Bollini - * @author Kostas Stamatis - * @author Luigi Andrea Pascarelli - * @author Panagiotis Koutsourakis - */ -public class ArXivService { - private int timeout = 1000; - - /** - * How long to wait for a connection to be established. - * - * @param timeout milliseconds - */ - public void setTimeout(int timeout) { - this.timeout = timeout; - } - - public List getByDOIs(Set dois) throws HttpException, - IOException { - if (dois != null && dois.size() > 0) { - String doisQuery = StringUtils.join(dois.iterator(), " OR "); - return search(doisQuery, null, 100); - } - return null; - } - - public List searchByTerm(String title, String author, int year) - throws HttpException, IOException { - StringBuffer query = new StringBuffer(); - if (StringUtils.isNotBlank(title)) { - query.append("ti:\"").append(title).append("\""); - } - if (StringUtils.isNotBlank(author)) { - // [FAU] - if (query.length() > 0) { - query.append(" AND "); - } - query.append("au:\"").append(author).append("\""); - } - return search(query.toString(), "", 10); - } - - protected List search(String query, String arxivid, int max_result) - throws IOException, HttpException { - List results = new ArrayList(); - HttpGet method = null; - try { - HttpClient client = new DefaultHttpClient(); - HttpParams params = client.getParams(); - params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, timeout); - - try { - URIBuilder uriBuilder = new URIBuilder("http://export.arxiv.org/api/query"); - uriBuilder.addParameter("id_list", arxivid); - uriBuilder.addParameter("search_query", query); - uriBuilder.addParameter("max_results", String.valueOf(max_result)); - method = new HttpGet(uriBuilder.build()); - } catch (URISyntaxException ex) { - throw new HttpException(ex.getMessage()); - } - - // Execute the method. - HttpResponse response = client.execute(method); - StatusLine responseStatus = response.getStatusLine(); - int statusCode = responseStatus.getStatusCode(); - - if (statusCode != HttpStatus.SC_OK) { - if (statusCode == HttpStatus.SC_BAD_REQUEST) { - throw new RuntimeException("arXiv query is not valid"); - } else { - throw new RuntimeException("Http call failed: " - + responseStatus); - } - } - - try { - DocumentBuilderFactory factory = DocumentBuilderFactory - .newInstance(); - factory.setValidating(false); - factory.setIgnoringComments(true); - factory.setIgnoringElementContentWhitespace(true); - - DocumentBuilder db = factory.newDocumentBuilder(); - Document inDoc = db.parse(response.getEntity().getContent()); - - Element xmlRoot = inDoc.getDocumentElement(); - List dataRoots = XMLUtils.getElementList(xmlRoot, - "entry"); - - for (Element dataRoot : dataRoots) { - Record crossitem = ArxivUtils - .convertArxixDomToRecord(dataRoot); - if (crossitem != null) { - results.add(crossitem); - } - } - } catch (Exception e) { - throw new RuntimeException( - "ArXiv identifier is not valid or not exist"); - } - } finally { - if (method != null) { - method.releaseConnection(); - } - } - - return results; - } - - public Record getByArXivIDs(String raw) throws HttpException, IOException { - if (StringUtils.isNotBlank(raw)) { - raw = raw.trim(); - if (raw.startsWith("http://arxiv.org/abs/")) { - raw = raw.substring("http://arxiv.org/abs/".length()); - } else if (raw.toLowerCase().startsWith("arxiv:")) { - raw = raw.substring("arxiv:".length()); - } - List result = search("", raw, 1); - if (result != null && result.size() > 0) { - return result.get(0); - } - } - return null; - } -} diff --git a/dspace-api/src/main/java/org/dspace/submit/lookup/ArxivUtils.java b/dspace-api/src/main/java/org/dspace/submit/lookup/ArxivUtils.java deleted file mode 100644 index 4caa0a957b..0000000000 --- a/dspace-api/src/main/java/org/dspace/submit/lookup/ArxivUtils.java +++ /dev/null @@ -1,151 +0,0 @@ -/** - * The contents of this file are subject to the license and copyright - * detailed in the LICENSE and NOTICE files at the root of the source - * tree and available online at - * - * http://www.dspace.org/license/ - */ -/** - * - */ -package org.dspace.submit.lookup; - -import java.util.LinkedList; -import java.util.List; - -import gr.ekt.bte.core.MutableRecord; -import gr.ekt.bte.core.Record; -import gr.ekt.bte.core.StringValue; -import gr.ekt.bte.core.Value; -import org.dspace.app.util.XMLUtils; -import org.dspace.submit.util.SubmissionLookupPublication; -import org.w3c.dom.Element; - -/** - * @author Andrea Bollini - * @author Kostas Stamatis - * @author Luigi Andrea Pascarelli - * @author Panagiotis Koutsourakis - */ -public class ArxivUtils { - - /** - * Default constructor - */ - private ArxivUtils() { } - - public static Record convertArxixDomToRecord(Element dataRoot) { - MutableRecord record = new SubmissionLookupPublication(""); - - String articleTitle = XMLUtils.getElementValue(dataRoot, "title"); - if (articleTitle != null) { - record.addValue("title", new StringValue(articleTitle)); - } - String summary = XMLUtils.getElementValue(dataRoot, "summary"); - if (summary != null) { - record.addValue("summary", new StringValue(summary)); - } - String year = XMLUtils.getElementValue(dataRoot, "published"); - if (year != null) { - record.addValue("published", new StringValue(year)); - } - String splashPageUrl = XMLUtils.getElementValue(dataRoot, "id"); - if (splashPageUrl != null) { - record.addValue("id", new StringValue(splashPageUrl)); - } - String comment = XMLUtils.getElementValue(dataRoot, "arxiv:comment"); - if (comment != null) { - record.addValue("comment", new StringValue(comment)); - } - - List links = XMLUtils.getElementList(dataRoot, "link"); - if (links != null) { - for (Element link : links) { - if ("related".equals(link.getAttribute("rel")) - && "pdf".equals(link.getAttribute("title"))) { - String pdfUrl = link.getAttribute("href"); - if (pdfUrl != null) { - record.addValue("pdfUrl", new StringValue(pdfUrl)); - } - } - } - } - - String doi = XMLUtils.getElementValue(dataRoot, "arxiv:doi"); - if (doi != null) { - record.addValue("doi", new StringValue(doi)); - } - String journalRef = XMLUtils.getElementValue(dataRoot, - "arxiv:journal_ref"); - if (journalRef != null) { - record.addValue("journalRef", new StringValue(journalRef)); - } - - List primaryCategory = new LinkedList(); - List primaryCategoryList = XMLUtils.getElementList(dataRoot, - "arxiv:primary_category"); - if (primaryCategoryList != null) { - for (Element primaryCategoryElement : primaryCategoryList) { - primaryCategory - .add(primaryCategoryElement.getAttribute("term")); - } - } - - if (primaryCategory.size() > 0) { - List values = new LinkedList(); - for (String s : primaryCategory) { - values.add(new StringValue(s)); - } - record.addField("primaryCategory", values); - } - - List category = new LinkedList(); - List categoryList = XMLUtils.getElementList(dataRoot, - "category"); - if (categoryList != null) { - for (Element categoryElement : categoryList) { - category.add(categoryElement.getAttribute("term")); - } - } - - if (category.size() > 0) { - List values = new LinkedList(); - for (String s : category) { - values.add(new StringValue(s)); - } - record.addField("category", values); - } - - List authors = new LinkedList(); - List authorsWithAffiliations = new LinkedList(); - List authorList = XMLUtils.getElementList(dataRoot, "author"); - if (authorList != null) { - for (Element authorElement : authorList) { - String authorName = XMLUtils.getElementValue(authorElement, "name"); - String authorAffiliation = XMLUtils.getElementValue(authorElement, "arxiv:affiliation"); - - authors.add(authorName); - authorsWithAffiliations.add(authorName + ": " + authorAffiliation); - } - } - - if (authors.size() > 0) { - List values = new LinkedList(); - for (String sArray : authors) { - values.add(new StringValue(sArray)); - } - record.addField("author", values); - } - - if (authorsWithAffiliations.size() > 0) { - List values = new LinkedList(); - for (String sArray : authorsWithAffiliations) { - values.add(new StringValue(sArray)); - } - record.addField("authorWithAffiliation", values); - } - - return record; - } - -} diff --git a/dspace-api/src/main/java/org/dspace/submit/lookup/PubmedFileDataLoader.java b/dspace-api/src/main/java/org/dspace/submit/lookup/PubmedFileDataLoader.java deleted file mode 100644 index 05a37e64d6..0000000000 --- a/dspace-api/src/main/java/org/dspace/submit/lookup/PubmedFileDataLoader.java +++ /dev/null @@ -1,148 +0,0 @@ -/** - * The contents of this file are subject to the license and copyright - * detailed in the LICENSE and NOTICE files at the root of the source - * tree and available online at - * - * http://www.dspace.org/license/ - */ - -package org.dspace.submit.lookup; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; -import java.util.Map; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; - -import gr.ekt.bte.core.DataLoadingSpec; -import gr.ekt.bte.core.Record; -import gr.ekt.bte.core.RecordSet; -import gr.ekt.bte.core.Value; -import gr.ekt.bte.dataloader.FileDataLoader; -import gr.ekt.bte.exceptions.MalformedSourceException; -import org.apache.commons.lang3.StringUtils; -import org.dspace.app.util.XMLUtils; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.xml.sax.SAXException; - -/** - * @author Andrea Bollini - * @author Kostas Stamatis - * @author Luigi Andrea Pascarelli - * @author Panagiotis Koutsourakis - */ -public class PubmedFileDataLoader extends FileDataLoader { - - Map fieldMap; // mapping between service fields and local - // intermediate fields - - /** - * - */ - public PubmedFileDataLoader() { - } - - /** - * @param filename Name of file to load CiNii data from. - */ - public PubmedFileDataLoader(String filename) { - super(filename); - } - - /* - * {@see gr.ekt.bte.core.DataLoader#getRecords()} - * - * @throws MalformedSourceException - */ - @Override - public RecordSet getRecords() throws MalformedSourceException { - - RecordSet recordSet = new RecordSet(); - - try { - InputStream inputStream = new FileInputStream(new File(filename)); - - DocumentBuilderFactory factory = DocumentBuilderFactory - .newInstance(); - factory.setValidating(false); - factory.setIgnoringComments(true); - factory.setIgnoringElementContentWhitespace(true); - - DocumentBuilder builder = factory.newDocumentBuilder(); - Document inDoc = builder.parse(inputStream); - - Element xmlRoot = inDoc.getDocumentElement(); - List pubArticles = XMLUtils.getElementList(xmlRoot, - "PubmedArticle"); - - for (Element xmlArticle : pubArticles) { - Record record = null; - try { - record = PubmedUtils.convertPubmedDomToRecord(xmlArticle); - recordSet.addRecord(convertFields(record)); - } catch (Exception e) { - throw new RuntimeException(e.getMessage(), e); - } - } - } catch (FileNotFoundException e) { - e.printStackTrace(); - } catch (ParserConfigurationException e) { - e.printStackTrace(); - } catch (SAXException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } - - return recordSet; - - } - - /* - * (non-Javadoc) - * - * @see - * gr.ekt.bte.core.DataLoader#getRecords(gr.ekt.bte.core.DataLoadingSpec) - */ - @Override - public RecordSet getRecords(DataLoadingSpec spec) - throws MalformedSourceException { - if (spec.getOffset() > 0) { - return new RecordSet(); - } - return getRecords(); - } - - public Record convertFields(Record publication) { - for (String fieldName : fieldMap.keySet()) { - String md = null; - if (fieldMap != null) { - md = this.fieldMap.get(fieldName); - } - - if (StringUtils.isBlank(md)) { - continue; - } else { - md = md.trim(); - } - - if (publication.isMutable()) { - List values = publication.getValues(fieldName); - publication.makeMutable().removeField(fieldName); - publication.makeMutable().addField(md, values); - } - } - - return publication; - } - - public void setFieldMap(Map fieldMap) { - this.fieldMap = fieldMap; - } -} diff --git a/dspace-api/src/main/java/org/dspace/submit/lookup/PubmedOnlineDataLoader.java b/dspace-api/src/main/java/org/dspace/submit/lookup/PubmedOnlineDataLoader.java deleted file mode 100644 index 094ce4e21d..0000000000 --- a/dspace-api/src/main/java/org/dspace/submit/lookup/PubmedOnlineDataLoader.java +++ /dev/null @@ -1,116 +0,0 @@ -/** - * The contents of this file are subject to the license and copyright - * detailed in the LICENSE and NOTICE files at the root of the source - * tree and available online at - * - * http://www.dspace.org/license/ - */ -package org.dspace.submit.lookup; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import gr.ekt.bte.core.Record; -import org.apache.http.HttpException; -import org.apache.logging.log4j.Logger; -import org.dspace.core.Context; -import org.dspace.core.LogManager; - -/** - * @author Andrea Bollini - * @author Kostas Stamatis - * @author Luigi Andrea Pascarelli - * @author Panagiotis Koutsourakis - */ -public class PubmedOnlineDataLoader extends NetworkSubmissionLookupDataLoader { - protected boolean searchProvider = true; - - private static final Logger log = org.apache.logging.log4j.LogManager.getLogger(PubmedOnlineDataLoader.class); - - protected PubmedService pubmedService = new PubmedService(); - - public void setPubmedService(PubmedService pubmedService) { - this.pubmedService = pubmedService; - } - - @Override - public List getSupportedIdentifiers() { - return Arrays.asList(new String[] {PUBMED, DOI}); - } - - public void setSearchProvider(boolean searchProvider) { - this.searchProvider = searchProvider; - } - - @Override - public boolean isSearchProvider() { - return searchProvider; - } - - @Override - public List getByIdentifier(Context context, - Map> keys) throws HttpException, IOException { - Set pmids = keys != null ? keys.get(PUBMED) : null; - Set dois = keys != null ? keys.get(DOI) : null; - List results = new ArrayList(); - if (pmids != null && pmids.size() > 0 - && (dois == null || dois.size() == 0)) { - for (String pmid : pmids) { - Record p = null; - try { - p = pubmedService.getByPubmedID(pmid); - } catch (Exception e) { - log.error(LogManager.getHeader(context, "getByIdentifier", - "pmid=" + pmid), e); - } - if (p != null) { - results.add(convertFields(p)); - } - } - } else if (dois != null && dois.size() > 0 - && (pmids == null || pmids.size() == 0)) { - StringBuffer query = new StringBuffer(); - for (String d : dois) { - if (query.length() > 0) { - query.append(" OR "); - } - query.append(d).append("[AI]"); - } - - List pubmedResults = pubmedService.search(query.toString()); - for (Record p : pubmedResults) { - results.add(convertFields(p)); - } - } else if (dois != null && dois.size() > 0 && pmids != null - && pmids.size() > 0) { - // EKT:ToDo: support list of dois and pmids in the search method of - // pubmedService - List pubmedResults = pubmedService.search(dois.iterator() - .next(), pmids.iterator().next()); - if (pubmedResults != null) { - for (Record p : pubmedResults) { - results.add(convertFields(p)); - } - } - } - - return results; - } - - @Override - public List search(Context context, String title, String author, - int year) throws HttpException, IOException { - List pubmedResults = pubmedService.search(title, author, year); - List results = new ArrayList(); - if (pubmedResults != null) { - for (Record p : pubmedResults) { - results.add(convertFields(p)); - } - } - return results; - } -} diff --git a/dspace-api/src/main/java/org/dspace/submit/lookup/PubmedService.java b/dspace-api/src/main/java/org/dspace/submit/lookup/PubmedService.java deleted file mode 100644 index fa30ee8ea5..0000000000 --- a/dspace-api/src/main/java/org/dspace/submit/lookup/PubmedService.java +++ /dev/null @@ -1,265 +0,0 @@ -/** - * The contents of this file are subject to the license and copyright - * detailed in the LICENSE and NOTICE files at the root of the source - * tree and available online at - * - * http://www.dspace.org/license/ - */ -package org.dspace.submit.lookup; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.List; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; - -import gr.ekt.bte.core.Record; -import org.apache.commons.lang3.StringUtils; -import org.apache.http.HttpException; -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.StatusLine; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.params.CoreConnectionPNames; -import org.apache.logging.log4j.Logger; -import org.dspace.app.util.XMLUtils; -import org.dspace.core.ConfigurationManager; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.xml.sax.SAXException; - -/** - * @author Andrea Bollini - * @author Kostas Stamatis - * @author Luigi Andrea Pascarelli - * @author Panagiotis Koutsourakis - */ -public class PubmedService { - - private static final Logger log = org.apache.logging.log4j.LogManager.getLogger(PubmedService.class); - - protected int timeout = 1000; - - public void setTimeout(int timeout) { - this.timeout = timeout; - } - - public Record getByPubmedID(String pubmedid) throws HttpException, - IOException, ParserConfigurationException, SAXException { - List ids = new ArrayList(); - ids.add(pubmedid.trim()); - List items = getByPubmedIDs(ids); - if (items != null && items.size() > 0) { - return items.get(0); - } - return null; - } - - public List search(String title, String author, int year) - throws HttpException, IOException { - StringBuffer query = new StringBuffer(); - if (StringUtils.isNotBlank(title)) { - query.append("((").append(title).append("[TI]) OR ("); - // [TI] does not always work, book chapter title - query.append("(").append(title).append("[book]))"); - } - if (StringUtils.isNotBlank(author)) { - // [FAU] - if (query.length() > 0) { - query.append(" AND "); - } - query.append("(").append(author).append("[AU])"); - } - if (year != -1) { - // [DP] - if (query.length() > 0) { - query.append(" AND "); - } - query.append(year).append("[DP]"); - } - return search(query.toString()); - } - - public List search(String query) throws IOException, HttpException { - List results = new ArrayList<>(); - if (!ConfigurationManager.getBooleanProperty(SubmissionLookupService.CFG_MODULE, "remoteservice.demo")) { - HttpGet method = null; - try { - HttpClient client = new DefaultHttpClient(); - client.getParams().setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, timeout); - - URIBuilder uriBuilder = new URIBuilder( - "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi"); - uriBuilder.addParameter("db", "pubmed"); - uriBuilder.addParameter("datetype", "edat"); - uriBuilder.addParameter("retmax", "10"); - uriBuilder.addParameter("term", query); - method = new HttpGet(uriBuilder.build()); - - // Execute the method. - HttpResponse response = client.execute(method); - StatusLine statusLine = response.getStatusLine(); - int statusCode = statusLine.getStatusCode(); - - if (statusCode != HttpStatus.SC_OK) { - throw new RuntimeException("WS call failed: " - + statusLine); - } - - DocumentBuilderFactory factory = DocumentBuilderFactory - .newInstance(); - factory.setValidating(false); - factory.setIgnoringComments(true); - factory.setIgnoringElementContentWhitespace(true); - - DocumentBuilder builder; - try { - builder = factory.newDocumentBuilder(); - - Document inDoc = builder.parse(response.getEntity().getContent()); - - Element xmlRoot = inDoc.getDocumentElement(); - Element idList = XMLUtils.getSingleElement(xmlRoot, - "IdList"); - List pubmedIDs = XMLUtils.getElementValueList( - idList, "Id"); - results = getByPubmedIDs(pubmedIDs); - } catch (ParserConfigurationException e1) { - log.error(e1.getMessage(), e1); - } catch (SAXException e1) { - log.error(e1.getMessage(), e1); - } - } catch (Exception e1) { - log.error(e1.getMessage(), e1); - } finally { - if (method != null) { - method.releaseConnection(); - } - } - } else { - InputStream stream = null; - try { - File file = new File( - ConfigurationManager.getProperty("dspace.dir") - + "/config/crosswalks/demo/pubmed-search.xml"); - stream = new FileInputStream(file); - DocumentBuilderFactory factory = DocumentBuilderFactory - .newInstance(); - factory.setValidating(false); - factory.setIgnoringComments(true); - factory.setIgnoringElementContentWhitespace(true); - - DocumentBuilder builder = factory.newDocumentBuilder(); - Document inDoc = builder.parse(stream); - - Element xmlRoot = inDoc.getDocumentElement(); - Element idList = XMLUtils.getSingleElement(xmlRoot, "IdList"); - List pubmedIDs = XMLUtils.getElementValueList(idList, - "Id"); - results = getByPubmedIDs(pubmedIDs); - } catch (Exception e) { - throw new RuntimeException(e.getMessage(), e); - } finally { - if (stream != null) { - try { - stream.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - } - return results; - } - - public List getByPubmedIDs(List pubmedIDs) - throws HttpException, IOException, ParserConfigurationException, - SAXException { - List results = new ArrayList(); - HttpGet method = null; - try { - HttpClient client = new DefaultHttpClient(); - client.getParams().setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 5 * timeout); - - try { - URIBuilder uriBuilder = new URIBuilder( - "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi"); - uriBuilder.addParameter("db", "pubmed"); - uriBuilder.addParameter("retmode", "xml"); - uriBuilder.addParameter("rettype", "full"); - uriBuilder.addParameter("id", StringUtils.join( - pubmedIDs.iterator(), ",")); - method = new HttpGet(uriBuilder.build()); - } catch (URISyntaxException ex) { - throw new RuntimeException("Request not sent", ex); - } - - // Execute the method. - HttpResponse response = client.execute(method); - StatusLine statusLine = response.getStatusLine(); - int statusCode = statusLine.getStatusCode(); - - if (statusCode != HttpStatus.SC_OK) { - throw new RuntimeException("WS call failed: " + statusLine); - } - - DocumentBuilderFactory factory = DocumentBuilderFactory - .newInstance(); - factory.setValidating(false); - factory.setIgnoringComments(true); - factory.setIgnoringElementContentWhitespace(true); - - DocumentBuilder builder = factory.newDocumentBuilder(); - Document inDoc = builder - .parse(response.getEntity().getContent()); - - Element xmlRoot = inDoc.getDocumentElement(); - List pubArticles = XMLUtils.getElementList(xmlRoot, - "PubmedArticle"); - - for (Element xmlArticle : pubArticles) { - Record pubmedItem = null; - try { - pubmedItem = PubmedUtils - .convertPubmedDomToRecord(xmlArticle); - results.add(pubmedItem); - } catch (Exception e) { - throw new RuntimeException( - "PubmedID is not valid or not exist: " - + e.getMessage(), e); - } - } - - return results; - } finally { - if (method != null) { - method.releaseConnection(); - } - } - } - - public List search(String doi, String pmid) throws HttpException, - IOException { - StringBuffer query = new StringBuffer(); - if (StringUtils.isNotBlank(doi)) { - query.append(doi); - query.append("[AID]"); - } - if (StringUtils.isNotBlank(pmid)) { - // [FAU] - if (query.length() > 0) { - query.append(" OR "); - } - query.append(pmid).append("[PMID]"); - } - return search(query.toString()); - } -} diff --git a/dspace-api/src/main/java/org/dspace/submit/lookup/PubmedUtils.java b/dspace-api/src/main/java/org/dspace/submit/lookup/PubmedUtils.java deleted file mode 100644 index bca34de295..0000000000 --- a/dspace-api/src/main/java/org/dspace/submit/lookup/PubmedUtils.java +++ /dev/null @@ -1,316 +0,0 @@ -/** - * The contents of this file are subject to the license and copyright - * detailed in the LICENSE and NOTICE files at the root of the source - * tree and available online at - * - * http://www.dspace.org/license/ - */ -/** - * - */ -package org.dspace.submit.lookup; - -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -import gr.ekt.bte.core.MutableRecord; -import gr.ekt.bte.core.Record; -import gr.ekt.bte.core.StringValue; -import gr.ekt.bte.core.Value; -import org.apache.commons.lang3.StringUtils; -import org.dspace.app.util.XMLUtils; -import org.dspace.submit.util.SubmissionLookupPublication; -import org.w3c.dom.Element; - -/** - * @author Andrea Bollini - * @author Kostas Stamatis - * @author Luigi Andrea Pascarelli - * @author Panagiotis Koutsourakis - */ -public class PubmedUtils { - - /** - * Default constructor - */ - private PubmedUtils() { } - - public static Record convertPubmedDomToRecord(Element pubArticle) { - MutableRecord record = new SubmissionLookupPublication(""); - - Map monthToNum = new HashMap(); - monthToNum.put("Jan", "01"); - monthToNum.put("Feb", "02"); - monthToNum.put("Mar", "03"); - monthToNum.put("Apr", "04"); - monthToNum.put("May", "05"); - monthToNum.put("Jun", "06"); - monthToNum.put("Jul", "07"); - monthToNum.put("Aug", "08"); - monthToNum.put("Sep", "09"); - monthToNum.put("Oct", "10"); - monthToNum.put("Nov", "11"); - monthToNum.put("Dec", "12"); - - Element medline = XMLUtils.getSingleElement(pubArticle, - "MedlineCitation"); - - Element article = XMLUtils.getSingleElement(medline, "Article"); - Element pubmed = XMLUtils.getSingleElement(pubArticle, "PubmedData"); - - Element identifierList = XMLUtils.getSingleElement(pubmed, - "ArticleIdList"); - if (identifierList != null) { - List identifiers = XMLUtils.getElementList(identifierList, - "ArticleId"); - if (identifiers != null) { - for (Element id : identifiers) { - if ("pubmed".equals(id.getAttribute("IdType"))) { - String pubmedID = id.getTextContent().trim(); - if (pubmedID != null) { - record.addValue("pubmedID", new StringValue( - pubmedID)); - } - } else if ("doi".equals(id.getAttribute("IdType"))) { - String doi = id.getTextContent().trim(); - if (doi != null) { - record.addValue("doi", new StringValue(doi)); - } - } - } - } - } - - String status = XMLUtils.getElementValue(pubmed, "PublicationStatus"); - if (status != null) { - record.addValue("publicationStatus", new StringValue(status)); - } - - String pubblicationModel = XMLUtils.getElementAttribute(medline, - "Article", "PubModel"); - if (pubblicationModel != null) { - record.addValue("pubModel", new StringValue( - pubblicationModel)); - } - - String title = XMLUtils.getElementValue(article, "ArticleTitle"); - if (title != null) { - record.addValue("articleTitle", new StringValue(title)); - } - - Element abstractElement = XMLUtils - .getSingleElement(article, "Abstract"); - if (abstractElement == null) { - abstractElement = XMLUtils.getSingleElement(medline, - "OtherAbstract"); - } - if (abstractElement != null) { - String summary = XMLUtils.getElementValue(abstractElement, - "AbstractText"); - if (summary != null) { - record.addValue("abstractText", new StringValue(summary)); - } - } - - List authors = new LinkedList(); - Element authorList = XMLUtils.getSingleElement(article, "AuthorList"); - if (authorList != null) { - List authorsElement = XMLUtils.getElementList(authorList, - "Author"); - if (authorsElement != null) { - for (Element author : authorsElement) { - if (StringUtils.isBlank(XMLUtils.getElementValue(author, - "CollectiveName"))) { - authors.add(new String[] { - XMLUtils.getElementValue(author, "ForeName"), - XMLUtils.getElementValue(author, "LastName")}); - } - } - } - } - if (authors.size() > 0) { - List values = new LinkedList(); - for (String[] sArray : authors) { - values.add(new StringValue(sArray[1] + ", " + sArray[0])); - } - record.addField("author", values); - } - - Element journal = XMLUtils.getSingleElement(article, "Journal"); - if (journal != null) { - List jnumbers = XMLUtils.getElementList(journal, "ISSN"); - if (jnumbers != null) { - for (Element jnumber : jnumbers) { - if ("Print".equals(jnumber.getAttribute("IssnType"))) { - String issn = jnumber.getTextContent().trim(); - if (issn != null) { - record.addValue("printISSN", new StringValue(issn)); - } - } else { - String eissn = jnumber.getTextContent().trim(); - if (eissn != null) { - record.addValue("electronicISSN", new StringValue(eissn)); - } - } - } - } - - String journalTitle = XMLUtils.getElementValue(journal, "Title"); - if (journalTitle != null) { - record.addValue("journalTitle", new StringValue(journalTitle)); - } - - Element journalIssueElement = XMLUtils.getSingleElement(journal, - "JournalIssue"); - if (journalIssueElement != null) { - String volume = XMLUtils.getElementValue(journalIssueElement, - "Volume"); - if (volume != null) { - record.addValue("journalVolume", new StringValue(volume)); - } - - String issue = XMLUtils.getElementValue(journalIssueElement, - "Issue"); - if (issue != null) { - record.addValue("journalIssue", new StringValue(issue)); - } - - Element pubDateElement = XMLUtils.getSingleElement( - journalIssueElement, "PubDate"); - - String pubDate = null; - if (pubDateElement != null) { - pubDate = XMLUtils.getElementValue(pubDateElement, "Year"); - - String mounth = XMLUtils.getElementValue(pubDateElement, - "Month"); - String day = XMLUtils - .getElementValue(pubDateElement, "Day"); - if (StringUtils.isNotBlank(mounth) - && monthToNum.containsKey(mounth)) { - pubDate += "-" + monthToNum.get(mounth); - if (StringUtils.isNotBlank(day)) { - pubDate += "-" + (day.length() == 1 ? "0" + day : day); - } - } - } - if (pubDate == null) { - pubDate = XMLUtils.getElementValue(pubDateElement, "MedlineDate"); - } - if (pubDate != null) { - record.addValue("pubDate", new StringValue(pubDate)); - } - } - - String language = XMLUtils.getElementValue(article, "Language"); - if (language != null) { - record.addValue("language", new StringValue(language)); - } - - List type = new LinkedList(); - Element publicationTypeList = XMLUtils.getSingleElement(article, - "PublicationTypeList"); - if (publicationTypeList != null) { - List publicationTypes = XMLUtils.getElementList( - publicationTypeList, "PublicationType"); - for (Element publicationType : publicationTypes) { - type.add(publicationType.getTextContent().trim()); - } - } - if (type.size() > 0) { - List values = new LinkedList(); - for (String s : type) { - values.add(new StringValue(s)); - } - record.addField("publicationType", values); - } - - List primaryKeywords = new LinkedList(); - List secondaryKeywords = new LinkedList(); - Element keywordsList = XMLUtils.getSingleElement(medline, - "KeywordList"); - if (keywordsList != null) { - List keywords = XMLUtils.getElementList(keywordsList, - "Keyword"); - for (Element keyword : keywords) { - if ("Y".equals(keyword.getAttribute("MajorTopicYN"))) { - primaryKeywords.add(keyword.getTextContent().trim()); - } else { - secondaryKeywords.add(keyword.getTextContent().trim()); - } - } - } - if (primaryKeywords.size() > 0) { - List values = new LinkedList(); - for (String s : primaryKeywords) { - values.add(new StringValue(s)); - } - record.addField("primaryKeyword", values); - } - if (secondaryKeywords.size() > 0) { - List values = new LinkedList(); - for (String s : secondaryKeywords) { - values.add(new StringValue(s)); - } - record.addField("secondaryKeyword", values); - } - - List primaryMeshHeadings = new LinkedList(); - List secondaryMeshHeadings = new LinkedList(); - Element meshHeadingsList = XMLUtils.getSingleElement(medline, - "MeshHeadingList"); - if (meshHeadingsList != null) { - List meshHeadings = XMLUtils.getElementList( - meshHeadingsList, "MeshHeading"); - for (Element meshHeading : meshHeadings) { - if ("Y".equals(XMLUtils.getElementAttribute(meshHeading, - "DescriptorName", "MajorTopicYN"))) { - primaryMeshHeadings.add(XMLUtils.getElementValue( - meshHeading, "DescriptorName")); - } else { - secondaryMeshHeadings.add(XMLUtils.getElementValue( - meshHeading, "DescriptorName")); - } - } - } - if (primaryMeshHeadings.size() > 0) { - List values = new LinkedList(); - for (String s : primaryMeshHeadings) { - values.add(new StringValue(s)); - } - record.addField("primaryMeshHeading", values); - } - if (secondaryMeshHeadings.size() > 0) { - List values = new LinkedList(); - for (String s : secondaryMeshHeadings) { - values.add(new StringValue(s)); - } - record.addField("secondaryMeshHeading", values); - } - - Element paginationElement = XMLUtils.getSingleElement(article, - "Pagination"); - if (paginationElement != null) { - String startPage = XMLUtils.getElementValue(paginationElement, - "StartPage"); - String endPage = XMLUtils.getElementValue(paginationElement, - "EndPage"); - if (StringUtils.isBlank(startPage)) { - startPage = XMLUtils.getElementValue(paginationElement, - "MedlinePgn"); - } - - if (startPage != null) { - record.addValue("startPage", new StringValue(startPage)); - } - if (endPage != null) { - record.addValue("endPage", new StringValue(endPage)); - } - } - } - - return record; - } -} diff --git a/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml b/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml index d2d6dd10c1..750e5eb309 100644 --- a/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml +++ b/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml @@ -36,7 +36,6 @@ - - - @@ -79,7 +77,6 @@ jeissn pisbn eisbn - arxivCategory keywords mesh language @@ -106,13 +103,9 @@ - - - - @@ -129,40 +122,11 @@ - - - - - - - - - - - arxivCategory - - - - - - - - - - - - - - publicationStatus - - - - @@ -357,75 +321,6 @@ value="http://ebooks.serrelib.gr/serrelib-oai/request" /> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -544,7 +439,6 @@ - @@ -553,7 +447,6 @@ - diff --git a/dspace/config/spring/api/step-processing-listener.xml b/dspace/config/spring/api/step-processing-listener.xml index eb016c5133..986b850875 100644 --- a/dspace/config/spring/api/step-processing-listener.xml +++ b/dspace/config/spring/api/step-processing-listener.xml @@ -13,9 +13,7 @@ - - From af38c36bdb094d91f934565d79bab66e29679363 Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Mon, 6 Jul 2020 11:33:52 +0200 Subject: [PATCH 24/96] add JavaDoc to LiveImportDataProvider public method --- .../provider/impl/LiveImportDataProvider.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/dspace-api/src/main/java/org/dspace/external/provider/impl/LiveImportDataProvider.java b/dspace-api/src/main/java/org/dspace/external/provider/impl/LiveImportDataProvider.java index 776984ddb7..7bc9766a19 100644 --- a/dspace-api/src/main/java/org/dspace/external/provider/impl/LiveImportDataProvider.java +++ b/dspace-api/src/main/java/org/dspace/external/provider/impl/LiveImportDataProvider.java @@ -47,18 +47,34 @@ public class LiveImportDataProvider implements ExternalDataProvider { return sourceIdentifier; } + /** + * This method set the SourceIdentifier for the ExternalDataProvider + * @param sourceIdentifier The UNIQUE sourceIdentifier to be set on any LiveImport data provider + */ public void setSourceIdentifier(String sourceIdentifier) { this.sourceIdentifier = sourceIdentifier; } + /** + * This method set the MetadataSource for the ExternalDataProvider + * @param metadataSource {@link org.dspace.importer.external.service.components.MetadataSource} implementation used to process the input data + */ public void setMetadataSource(MetadataSource metadataSource) { this.metadataSource = metadataSource; } + /** + * This method set dublin core identifier to use as metadata id + * @param recordIdMetadata dublin core identifier to use as metadata id + */ public void setRecordIdMetadata(String recordIdMetadata) { this.recordIdMetadata = recordIdMetadata; } + /** + * This method set the dublin core identifier to display the title + * @param displayMetadata metadata to use as title + */ public void setDisplayMetadata(String displayMetadata) { this.displayMetadata = displayMetadata; } From 56be1bbe745dd0a6202e31e27173f9e1110cc46e Mon Sep 17 00:00:00 2001 From: Marie Verdonck Date: Wed, 8 Jul 2020 14:19:54 +0200 Subject: [PATCH 25/96] 71734: Sitemap REST controller --- .../app/rest/SitemapRestController.java | 148 ++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 dspace-server-webapp/src/main/java/org/dspace/app/rest/SitemapRestController.java diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/SitemapRestController.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/SitemapRestController.java new file mode 100644 index 0000000000..310e2e5a39 --- /dev/null +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/SitemapRestController.java @@ -0,0 +1,148 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.sql.SQLException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import javassist.NotFoundException; +import org.apache.catalina.connector.ClientAbortException; +import org.apache.logging.log4j.Logger; +import org.dspace.app.rest.model.RestModel; +import org.dspace.app.rest.utils.ContextUtil; +import org.dspace.app.rest.utils.MultipartFileSender; +import org.dspace.core.Context; +import org.dspace.services.ConfigurationService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +/** + * This is a specialized controller to provide access to the sitemap files, generated by + * {@link org.dspace.app.sitemap.GenerateSitemaps} + * + * The mapping for requested endpoint try to resolve a valid sitemap file name, for example + * 
+ * {@code
+ * https:///api/discover/sitemaps/26453b4d-e513-44e8-8d5b-395f62972eff/sitemap0.html
+ * }
+ *
+ * + * @author Maria Verdonck (Atmire) on 08/07/2020 + */ +@RestController +@RequestMapping("/api/" + RestModel.DISCOVER + "/sitemaps") +public class SitemapRestController { + + private static final Logger log = org.apache.logging.log4j.LogManager.getLogger(SitemapRestController.class); + + @Autowired + ConfigurationService configurationService; + + // Most file systems are configured to use block sizes of 4096 or 8192 and our buffer should be a multiple of that. + private static final int BUFFER_SIZE = 4096 * 10; + + /** + * Tries to retrieve a matching sitemap file in configured location + * + * @param name the name of the requested sitemap file + * @param response the HTTP response + * @param request the HTTP request + * @throws NotFoundException if no matching sitemap file can be found + * @throws SQLException if db error while completing DSpace context + * @throws IOException if IO error surrounding sitemap file + */ + @RequestMapping(method = {RequestMethod.GET, RequestMethod.HEAD}, value = "/{name}") + public void retrieve(@PathVariable String name, HttpServletResponse response, + HttpServletRequest request) throws NotFoundException, IOException, SQLException { + // Find sitemap with given name in dspace/sitemaps + File foundSitemapFile = null; + File sitemapOutputDir = new File(configurationService.getProperty("sitemap.dir")); + if (sitemapOutputDir.exists() && sitemapOutputDir.isDirectory()) { + // List of all files and directories inside sitemapOutputDir + File sitemapFilesList[] = sitemapOutputDir.listFiles(); + for (File sitemapFile : sitemapFilesList) { + if (name.equalsIgnoreCase(sitemapFile.getName())) { + if (sitemapFile.isFile()) { + foundSitemapFile = sitemapFile; + } else { + throw new NotFoundException( + "Directory with name " + name + " in " + sitemapOutputDir.getAbsolutePath() + + " found, but no file"); + } + } + } + } else { + throw new NotFoundException("Sitemap directory in " + sitemapOutputDir.getAbsolutePath() + " does not " + + "exist, either sitemaps have not been generated, or are located elsewhere."); + } + if (foundSitemapFile == null) { + throw new NotFoundException( + "Could not find sitemap file with name " + name + " in " + sitemapOutputDir.getAbsolutePath()); + } else { + // return found sitemap file + this.returnSitemapFile(foundSitemapFile, response, request); + } + } + + /** + * Sends back the matching sitemap file as a MultipartFile, with the headers set with details of the file + * (content, size, name, last modified) + * + * @param foundSitemapFile the found sitemap file, with matching name as in request path + * @param response the HTTP response + * @param request the HTTP request + * @throws SQLException if db error while completing DSpace context + * @throws IOException if IO error surrounding sitemap file + */ + private void returnSitemapFile(File foundSitemapFile, HttpServletResponse response, HttpServletRequest request) + throws SQLException, IOException { + // Pipe the bits + try (InputStream is = new FileInputStream(foundSitemapFile)) { + MultipartFileSender sender = MultipartFileSender + .fromInputStream(is) + .withBufferSize(BUFFER_SIZE) + .withFileName(foundSitemapFile.getName()) + .withLength(foundSitemapFile.length()) + .withMimetype(Files.probeContentType(foundSitemapFile.toPath())) + .with(request) + .with(response); + + sender.withLastModified(foundSitemapFile.lastModified()); + + // Determine if we need to send the file as a download or if the browser can open it inline + long dispositionThreshold = configurationService.getLongProperty("webui.content_disposition_threshold"); + if (dispositionThreshold >= 0 && foundSitemapFile.length() > dispositionThreshold) { + sender.withDisposition(MultipartFileSender.CONTENT_DISPOSITION_ATTACHMENT); + } + + Context context = ContextUtil.obtainContext(request); + + // We have all the data we need, close the connection to the database so that it doesn't stay open during + // download/streaming + context.complete(); + + // Send the data + if (sender.isValid()) { + sender.serveResource(); + } + + } catch (ClientAbortException e) { + log.debug("Client aborted the request before the download was completed. " + + "Client is probably switching to a Range request.", e); + } + } +} From 894738a6826a93719b419741a810eb819fdf7e6f Mon Sep 17 00:00:00 2001 From: Marie Verdonck Date: Wed, 8 Jul 2020 15:44:58 +0200 Subject: [PATCH 26/96] 71736: Changes to sitemap contents --- .../dspace/app/sitemap/GenerateSitemaps.java | 23 ++++++++----------- .../app/sitemap/SitemapsOrgGenerator.java | 6 ++--- .../app/rest/SitemapRestController.java | 6 ++--- dspace/config/dspace.cfg | 2 +- 4 files changed, 17 insertions(+), 20 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java b/dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java index bb35cd3ff9..99517773a5 100644 --- a/dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java +++ b/dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java @@ -61,6 +61,8 @@ public class GenerateSitemaps { private static final ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService(); + public static final String SITEMAPS_ENDPOINT = "/api/discover/sitemaps"; + /** * Default constructor */ @@ -152,12 +154,9 @@ public class GenerateSitemaps { */ public static void generateSitemaps(boolean makeHTMLMap, boolean makeSitemapOrg) throws SQLException, IOException { - String sitemapStem = configurationService.getProperty("dspace.ui.url") - + "/sitemap"; - String htmlMapStem = configurationService.getProperty("dspace.ui.url") - + "/htmlmap"; - String handleURLStem = configurationService.getProperty("dspace.ui.url") - + "/handle/"; + String sitemapStem = configurationService.getProperty("dspace.server.url") + + SITEMAPS_ENDPOINT + "/sitemap"; + String uiURLStem = configurationService.getProperty("dspace.ui.url"); File outputDir = new File(configurationService.getProperty("sitemap.dir")); if (!outputDir.exists() && !outputDir.mkdir()) { @@ -168,13 +167,11 @@ public class GenerateSitemaps { AbstractGenerator sitemapsOrg = null; if (makeHTMLMap) { - html = new HTMLSitemapGenerator(outputDir, htmlMapStem + "?map=", - null); + html = new HTMLSitemapGenerator(outputDir, sitemapStem, ".html"); } if (makeSitemapOrg) { - sitemapsOrg = new SitemapsOrgGenerator(outputDir, sitemapStem - + "?map=", null); + sitemapsOrg = new SitemapsOrgGenerator(outputDir, sitemapStem, ".xml"); } Context c = new Context(Context.Mode.READ_ONLY); @@ -182,7 +179,7 @@ public class GenerateSitemaps { List comms = communityService.findAll(c); for (Community comm : comms) { - String url = handleURLStem + comm.getHandle(); + String url = uiURLStem + "/communities/" + comm.getID(); if (makeHTMLMap) { html.addURL(url, null); @@ -197,7 +194,7 @@ public class GenerateSitemaps { List colls = collectionService.findAll(c); for (Collection coll : colls) { - String url = handleURLStem + coll.getHandle(); + String url = uiURLStem + "/collections/" + coll.getID(); if (makeHTMLMap) { html.addURL(url, null); @@ -214,7 +211,7 @@ public class GenerateSitemaps { while (allItems.hasNext()) { Item i = allItems.next(); - String url = handleURLStem + i.getHandle(); + String url = uiURLStem + "/items/" + i.getID(); Date lastMod = i.getLastModified(); if (makeHTMLMap) { diff --git a/dspace-api/src/main/java/org/dspace/app/sitemap/SitemapsOrgGenerator.java b/dspace-api/src/main/java/org/dspace/app/sitemap/SitemapsOrgGenerator.java index 9a0d5a6ba4..3ec4ca8239 100644 --- a/dspace-api/src/main/java/org/dspace/app/sitemap/SitemapsOrgGenerator.java +++ b/dspace-api/src/main/java/org/dspace/app/sitemap/SitemapsOrgGenerator.java @@ -59,7 +59,7 @@ public class SitemapsOrgGenerator extends AbstractGenerator { @Override public String getFilename(int number) { - return "sitemap" + number + ".xml.gz"; + return "sitemap" + number + ".xml"; } @Override @@ -100,12 +100,12 @@ public class SitemapsOrgGenerator extends AbstractGenerator { @Override public boolean useCompression() { - return true; + return false; } @Override public String getIndexFilename() { - return "sitemap_index.xml.gz"; + return "sitemap_index.xml"; } @Override diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/SitemapRestController.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/SitemapRestController.java index 310e2e5a39..dba5d2e46e 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/SitemapRestController.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/SitemapRestController.java @@ -19,9 +19,9 @@ import javax.servlet.http.HttpServletResponse; import javassist.NotFoundException; import org.apache.catalina.connector.ClientAbortException; import org.apache.logging.log4j.Logger; -import org.dspace.app.rest.model.RestModel; import org.dspace.app.rest.utils.ContextUtil; import org.dspace.app.rest.utils.MultipartFileSender; +import org.dspace.app.sitemap.GenerateSitemaps; import org.dspace.core.Context; import org.dspace.services.ConfigurationService; import org.springframework.beans.factory.annotation.Autowired; @@ -44,7 +44,7 @@ import org.springframework.web.bind.annotation.RestController; * @author Maria Verdonck (Atmire) on 08/07/2020 */ @RestController -@RequestMapping("/api/" + RestModel.DISCOVER + "/sitemaps") +@RequestMapping(GenerateSitemaps.SITEMAPS_ENDPOINT) public class SitemapRestController { private static final Logger log = org.apache.logging.log4j.LogManager.getLogger(SitemapRestController.class); @@ -81,7 +81,7 @@ public class SitemapRestController { } else { throw new NotFoundException( "Directory with name " + name + " in " + sitemapOutputDir.getAbsolutePath() + - " found, but no file"); + " found, but no file."); } } } diff --git a/dspace/config/dspace.cfg b/dspace/config/dspace.cfg index 5090f5ea34..4311887f96 100644 --- a/dspace/config/dspace.cfg +++ b/dspace/config/dspace.cfg @@ -28,7 +28,7 @@ dspace.server.url = http://localhost:8080/server # URL of DSpace frontend (Angular UI). Include port number etc # This is used by the backend to provide links in emails, RSS feeds, Sitemaps, etc. -dspace.ui.url = http://localhost:3000 +dspace.ui.url = http://localhost:4000 # Name of the site dspace.name = DSpace at My University From 0480b6521948b952cf2d85bcc3a901fd5a5bda82 Mon Sep 17 00:00:00 2001 From: Marie Verdonck Date: Wed, 8 Jul 2020 17:05:36 +0200 Subject: [PATCH 27/96] 71737: Sitemap REST IT --- .../app/rest/SitemapRestController.java | 23 +-- .../app/rest/utils/MultipartFileSender.java | 10 +- .../rest/RootRestResourceControllerIT.java | 9 +- .../app/rest/ShibbolethRestControllerIT.java | 7 +- .../app/rest/SitemapRestControllerIT.java | 142 ++++++++++++++++++ 5 files changed, 174 insertions(+), 17 deletions(-) create mode 100644 dspace-server-webapp/src/test/java/org/dspace/app/rest/SitemapRestControllerIT.java diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/SitemapRestController.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/SitemapRestController.java index dba5d2e46e..41a0c96ffa 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/SitemapRestController.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/SitemapRestController.java @@ -16,15 +16,15 @@ import java.sql.SQLException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import javassist.NotFoundException; import org.apache.catalina.connector.ClientAbortException; import org.apache.logging.log4j.Logger; +import org.dspace.app.rest.model.RestModel; import org.dspace.app.rest.utils.ContextUtil; import org.dspace.app.rest.utils.MultipartFileSender; -import org.dspace.app.sitemap.GenerateSitemaps; import org.dspace.core.Context; import org.dspace.services.ConfigurationService; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.rest.webmvc.ResourceNotFoundException; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; @@ -44,7 +44,7 @@ import org.springframework.web.bind.annotation.RestController; * @author Maria Verdonck (Atmire) on 08/07/2020 */ @RestController -@RequestMapping(GenerateSitemaps.SITEMAPS_ENDPOINT) +@RequestMapping("/api/" + RestModel.DISCOVER + "/sitemaps") public class SitemapRestController { private static final Logger log = org.apache.logging.log4j.LogManager.getLogger(SitemapRestController.class); @@ -61,13 +61,12 @@ public class SitemapRestController { * @param name the name of the requested sitemap file * @param response the HTTP response * @param request the HTTP request - * @throws NotFoundException if no matching sitemap file can be found - * @throws SQLException if db error while completing DSpace context - * @throws IOException if IO error surrounding sitemap file + * @throws SQLException if db error while completing DSpace context + * @throws IOException if IO error surrounding sitemap file */ @RequestMapping(method = {RequestMethod.GET, RequestMethod.HEAD}, value = "/{name}") public void retrieve(@PathVariable String name, HttpServletResponse response, - HttpServletRequest request) throws NotFoundException, IOException, SQLException { + HttpServletRequest request) throws IOException, SQLException { // Find sitemap with given name in dspace/sitemaps File foundSitemapFile = null; File sitemapOutputDir = new File(configurationService.getProperty("sitemap.dir")); @@ -79,18 +78,20 @@ public class SitemapRestController { if (sitemapFile.isFile()) { foundSitemapFile = sitemapFile; } else { - throw new NotFoundException( + throw new ResourceNotFoundException( "Directory with name " + name + " in " + sitemapOutputDir.getAbsolutePath() + " found, but no file."); } } } } else { - throw new NotFoundException("Sitemap directory in " + sitemapOutputDir.getAbsolutePath() + " does not " + - "exist, either sitemaps have not been generated, or are located elsewhere."); + throw new ResourceNotFoundException( + "Sitemap directory in " + sitemapOutputDir.getAbsolutePath() + " does not " + + "exist, either sitemaps have not been generated (./dspace generate-sitemaps)," + + " or are located elsewhere (config used: sitemap.dir)."); } if (foundSitemapFile == null) { - throw new NotFoundException( + throw new ResourceNotFoundException( "Could not find sitemap file with name " + name + " in " + sitemapOutputDir.getAbsolutePath()); } else { // return found sitemap file diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/utils/MultipartFileSender.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/utils/MultipartFileSender.java index 4ae836bccf..284d0b87ab 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/utils/MultipartFileSender.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/utils/MultipartFileSender.java @@ -156,9 +156,13 @@ public class MultipartFileSender { // Initialize response. response.reset(); response.setBufferSize(bufferSize); - response.setHeader(CONTENT_TYPE, contentType); + if (contentType != null) { + response.setHeader(CONTENT_TYPE, contentType); + } response.setHeader(ACCEPT_RANGES, BYTES); - response.setHeader(ETAG, checksum); + if (checksum != null) { + response.setHeader(ETAG, checksum); + } response.setDateHeader(LAST_MODIFIED, lastModified); response.setDateHeader(EXPIRES, System.currentTimeMillis() + DEFAULT_EXPIRE_TIME); @@ -481,4 +485,4 @@ public class MultipartFileSender { return Arrays.binarySearch(matchValues, toMatch) > -1 || Arrays.binarySearch(matchValues, "*") > -1; } -} \ No newline at end of file +} diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/RootRestResourceControllerIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/RootRestResourceControllerIT.java index 89ed50ec24..6ad1d23b18 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/RootRestResourceControllerIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/RootRestResourceControllerIT.java @@ -14,8 +14,10 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import org.dspace.app.rest.test.AbstractControllerIntegrationTest; +import org.dspace.services.ConfigurationService; import org.hamcrest.Matchers; import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; /** * Integration test for the {@link RootRestResourceController} @@ -25,6 +27,9 @@ import org.junit.Test; */ public class RootRestResourceControllerIT extends AbstractControllerIntegrationTest { + @Autowired + ConfigurationService configurationService; + @Test public void serverPropertiesTest() throws Exception { //When we call the root endpoint @@ -33,7 +38,7 @@ public class RootRestResourceControllerIT extends AbstractControllerIntegrationT .andExpect(status().isOk()) //We expect the content type to be "application/hal+json;charset=UTF-8" .andExpect(content().contentType(contentType)) - .andExpect(jsonPath("$.dspaceURL", Matchers.is("http://localhost:3000"))) + .andExpect(jsonPath("$.dspaceURL", Matchers.is(configurationService.getProperty("dspace.ui.url")))) .andExpect(jsonPath("$.dspaceName", Matchers.is("DSpace at My University"))) .andExpect(jsonPath("$.dspaceRest", Matchers.is(BASE_REST_SERVER_URL))) .andExpect(jsonPath("$.type", Matchers.is("root"))); @@ -71,4 +76,4 @@ public class RootRestResourceControllerIT extends AbstractControllerIntegrationT ; } -} \ No newline at end of file +} diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ShibbolethRestControllerIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ShibbolethRestControllerIT.java index a690e539ab..eff30a14c0 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ShibbolethRestControllerIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ShibbolethRestControllerIT.java @@ -12,7 +12,9 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import org.dspace.app.rest.test.AbstractControllerIntegrationTest; +import org.dspace.services.ConfigurationService; import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; /** * Integration test that cover ShibbolethRestController @@ -21,13 +23,16 @@ import org.junit.Test; */ public class ShibbolethRestControllerIT extends AbstractControllerIntegrationTest { + @Autowired + ConfigurationService configurationService; + @Test public void testRedirectToDefaultDspaceUrl() throws Exception { String token = getAuthToken(eperson.getEmail(), password); getClient(token).perform(get("/api/authn/shibboleth")) .andExpect(status().is3xxRedirection()) - .andExpect(redirectedUrl("http://localhost:3000")); + .andExpect(redirectedUrl(configurationService.getProperty("dspace.ui.url"))); } @Test diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/SitemapRestControllerIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/SitemapRestControllerIT.java new file mode 100644 index 0000000000..7ef453d22d --- /dev/null +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/SitemapRestControllerIT.java @@ -0,0 +1,142 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest; + +import static org.dspace.app.rest.builder.CollectionBuilder.createCollection; +import static org.dspace.app.rest.builder.CommunityBuilder.createCommunity; +import static org.dspace.app.rest.builder.ItemBuilder.createItem; +import static org.junit.Assert.assertTrue; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import org.dspace.app.rest.model.RestModel; +import org.dspace.app.rest.test.AbstractControllerIntegrationTest; +import org.dspace.content.Collection; +import org.dspace.content.Community; +import org.dspace.content.Item; +import org.dspace.services.ConfigurationService; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.web.servlet.MvcResult; + +/** + * Integration test to test the /api/discover/sitemaps/{name} endpoint, see {@link SitemapRestController} + * + * @author Maria Verdonck (Atmire) on 08/07/2020 + */ +public class SitemapRestControllerIT extends AbstractControllerIntegrationTest { + + @Autowired + ConfigurationService configurationService; + + private final static String SITEMAPS_ENDPOINT = "/api/" + RestModel.DISCOVER + "/sitemaps"; + + private Item item1; + private Item item2; + + @Before + @Override + public void setUp() throws Exception { + + super.setUp(); + + context.turnOffAuthorisationSystem(); + + Community community = createCommunity(context).build(); + Collection collection = createCollection(context, community).build(); + this.item1 = createItem(context, collection) + .withTitle("Test 1") + .withIssueDate("2010-10-17") + .build(); + this.item2 = createItem(context, collection) + .withTitle("Test 2") + .withIssueDate("2015-8-3") + .build(); + + runDSpaceScript("generate-sitemaps"); + + context.restoreAuthSystemState(); + } + + @Test + public void testSitemap_notValidSiteMapFile() throws Exception { + //** WHEN ** + //We attempt to retrieve a non valid sitemap file + getClient().perform(get(SITEMAPS_ENDPOINT + "/notValidSiteMapFile")) + //** THEN ** + .andExpect(status().isNotFound()); + } + + @Test + public void testSitemap_sitemapIndexHtml() throws Exception { + //** WHEN ** + //We retrieve sitemap_index.html + MvcResult result = getClient().perform(get(SITEMAPS_ENDPOINT + "/sitemap_index.html")) + //** THEN ** + .andExpect(status().isOk()) + //We expect the content type to match + .andExpect(content().contentType("text/html")) + .andReturn(); + + String response = result.getResponse().getContentAsString(); + // contains a link to /api/discover/sitemaps/sitemap0.html + assertTrue(response.contains(SITEMAPS_ENDPOINT + "/sitemap0.html")); + } + + @Test + public void testSitemap_sitemap0Html() throws Exception { + //** WHEN ** + //We retrieve sitemap0.html + MvcResult result = getClient().perform(get(SITEMAPS_ENDPOINT + "/sitemap0.html")) + //** THEN ** + .andExpect(status().isOk()) + //We expect the content type to match + .andExpect(content().contentType("text/html")) + .andReturn(); + + String response = result.getResponse().getContentAsString(); + // contains a link to items: [dspace.ui.url]/items/ + assertTrue(response.contains(configurationService.getProperty("dspace.ui.url") + "/items/" + item1.getID())); + assertTrue(response.contains(configurationService.getProperty("dspace.ui.url") + "/items/" + item2.getID())); + } + + @Test + public void testSitemap_sitemapIndexXml() throws Exception { + //** WHEN ** + //We retrieve sitemap_index.xml + MvcResult result = getClient().perform(get(SITEMAPS_ENDPOINT + "/sitemap_index.xml")) + //** THEN ** + .andExpect(status().isOk()) + //We expect the content type to match + .andExpect(content().contentType("application/xml")) + .andReturn(); + + String response = result.getResponse().getContentAsString(); + // contains a link to /api/discover/sitemaps/sitemap0.html + assertTrue(response.contains(SITEMAPS_ENDPOINT + "/sitemap0.xml")); + } + + @Test + public void testSitemap_sitemap0Xml() throws Exception { + //** WHEN ** + //We retrieve sitemap0.html + MvcResult result = getClient().perform(get(SITEMAPS_ENDPOINT + "/sitemap0.xml")) + //** THEN ** + .andExpect(status().isOk()) + //We expect the content type to match + .andExpect(content().contentType("application/xml")) + .andReturn(); + + String response = result.getResponse().getContentAsString(); + // contains a link to items: [dspace.ui.url]/items/ + assertTrue(response.contains(configurationService.getProperty("dspace.ui.url") + "/items/" + item1.getID())); + assertTrue(response.contains(configurationService.getProperty("dspace.ui.url") + "/items/" + item2.getID())); + } +} From d0a7104e9f5747cf235d254b6a0efec3b13e38b1 Mon Sep 17 00:00:00 2001 From: Marie Verdonck Date: Thu, 9 Jul 2020 10:52:05 +0200 Subject: [PATCH 28/96] 71738: Automatic cron job for sitemaps --- .../dspace/app/sitemap/GenerateSitemaps.java | 13 ++++++++++-- dspace/config/spring/api/task-scheduler.xml | 20 +++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 dspace/config/spring/api/task-scheduler.xml diff --git a/dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java b/dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java index 99517773a5..2a4698a833 100644 --- a/dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java +++ b/dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java @@ -142,6 +142,16 @@ public class GenerateSitemaps { System.exit(0); } + /** + * Runs generate-sitemaps without any params for the scheduler (task-scheduler.xml). + * + * @throws SQLException if a database error occurs. + * @throws IOException if IO error occurs. + */ + public static void generateSitemapsScheduled() throws IOException, SQLException { + generateSitemaps(true, true); + } + /** * Generate sitemap.org protocol and/or basic HTML sitemaps. * @@ -152,8 +162,7 @@ public class GenerateSitemaps { * @throws IOException if IO error * if IO error occurs. */ - public static void generateSitemaps(boolean makeHTMLMap, - boolean makeSitemapOrg) throws SQLException, IOException { + public static void generateSitemaps(boolean makeHTMLMap, boolean makeSitemapOrg) throws SQLException, IOException { String sitemapStem = configurationService.getProperty("dspace.server.url") + SITEMAPS_ENDPOINT + "/sitemap"; String uiURLStem = configurationService.getProperty("dspace.ui.url"); diff --git a/dspace/config/spring/api/task-scheduler.xml b/dspace/config/spring/api/task-scheduler.xml new file mode 100644 index 0000000000..a77b7aafaa --- /dev/null +++ b/dspace/config/spring/api/task-scheduler.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + From c911a0f79a61cfed9624c983e849b69ccfbc86fa Mon Sep 17 00:00:00 2001 From: Marie Verdonck Date: Wed, 15 Jul 2020 13:27:55 +0200 Subject: [PATCH 29/96] 71859: Endpoint path sitemaps changed --- .../org/dspace/app/sitemap/GenerateSitemaps.java | 2 +- .../org/dspace/app/rest/SitemapRestController.java | 13 ++++++------- .../dspace/app/rest/SitemapRestControllerIT.java | 9 ++++----- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java b/dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java index 2a4698a833..907d34a47b 100644 --- a/dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java +++ b/dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java @@ -61,7 +61,7 @@ public class GenerateSitemaps { private static final ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService(); - public static final String SITEMAPS_ENDPOINT = "/api/discover/sitemaps"; + public static final String SITEMAPS_ENDPOINT = "/sitemaps"; /** * Default constructor diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/SitemapRestController.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/SitemapRestController.java index 41a0c96ffa..f9e4281f51 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/SitemapRestController.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/SitemapRestController.java @@ -18,17 +18,16 @@ import javax.servlet.http.HttpServletResponse; import org.apache.catalina.connector.ClientAbortException; import org.apache.logging.log4j.Logger; -import org.dspace.app.rest.model.RestModel; import org.dspace.app.rest.utils.ContextUtil; import org.dspace.app.rest.utils.MultipartFileSender; import org.dspace.core.Context; import org.dspace.services.ConfigurationService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.rest.webmvc.ResourceNotFoundException; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.RestController; /** * This is a specialized controller to provide access to the sitemap files, generated by @@ -37,14 +36,14 @@ import org.springframework.web.bind.annotation.RestController; * The mapping for requested endpoint try to resolve a valid sitemap file name, for example * 
  * {@code
- * https:///api/discover/sitemaps/26453b4d-e513-44e8-8d5b-395f62972eff/sitemap0.html
+ * https:///sitemaps/26453b4d-e513-44e8-8d5b-395f62972eff/sitemap0.html
  * }
  *
* * @author Maria Verdonck (Atmire) on 08/07/2020 */ -@RestController -@RequestMapping("/api/" + RestModel.DISCOVER + "/sitemaps") +@Controller +@RequestMapping("/sitemaps") public class SitemapRestController { private static final Logger log = org.apache.logging.log4j.LogManager.getLogger(SitemapRestController.class); @@ -64,7 +63,7 @@ public class SitemapRestController { * @throws SQLException if db error while completing DSpace context * @throws IOException if IO error surrounding sitemap file */ - @RequestMapping(method = {RequestMethod.GET, RequestMethod.HEAD}, value = "/{name}") + @GetMapping("/{name}") public void retrieve(@PathVariable String name, HttpServletResponse response, HttpServletRequest request) throws IOException, SQLException { // Find sitemap with given name in dspace/sitemaps diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/SitemapRestControllerIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/SitemapRestControllerIT.java index 7ef453d22d..032de65ac7 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/SitemapRestControllerIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/SitemapRestControllerIT.java @@ -15,7 +15,6 @@ import static org.springframework.test.web.servlet.request.MockMvcRequestBuilder import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import org.dspace.app.rest.model.RestModel; import org.dspace.app.rest.test.AbstractControllerIntegrationTest; import org.dspace.content.Collection; import org.dspace.content.Community; @@ -27,7 +26,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.web.servlet.MvcResult; /** - * Integration test to test the /api/discover/sitemaps/{name} endpoint, see {@link SitemapRestController} + * Integration test to test the /sitemaps/{name} endpoint, see {@link SitemapRestController} * * @author Maria Verdonck (Atmire) on 08/07/2020 */ @@ -36,7 +35,7 @@ public class SitemapRestControllerIT extends AbstractControllerIntegrationTest { @Autowired ConfigurationService configurationService; - private final static String SITEMAPS_ENDPOINT = "/api/" + RestModel.DISCOVER + "/sitemaps"; + private final static String SITEMAPS_ENDPOINT = "/sitemaps"; private Item item1; private Item item2; @@ -86,7 +85,7 @@ public class SitemapRestControllerIT extends AbstractControllerIntegrationTest { .andReturn(); String response = result.getResponse().getContentAsString(); - // contains a link to /api/discover/sitemaps/sitemap0.html + // contains a link to /sitemaps/sitemap0.html assertTrue(response.contains(SITEMAPS_ENDPOINT + "/sitemap0.html")); } @@ -119,7 +118,7 @@ public class SitemapRestControllerIT extends AbstractControllerIntegrationTest { .andReturn(); String response = result.getResponse().getContentAsString(); - // contains a link to /api/discover/sitemaps/sitemap0.html + // contains a link to /sitemaps/sitemap0.html assertTrue(response.contains(SITEMAPS_ENDPOINT + "/sitemap0.xml")); } From 76d2e067108a3b7099e5e901cf69d47bb6d71396 Mon Sep 17 00:00:00 2001 From: Mykhaylo Date: Mon, 20 Jul 2020 17:53:07 +0200 Subject: [PATCH 30/96] added Logger --- .../contributor/ArXivIdMetadataContributor.java | 8 ++++++-- .../contributor/SimpleXpathMetadatumContributor.java | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/contributor/ArXivIdMetadataContributor.java b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/contributor/ArXivIdMetadataContributor.java index 727a3fcf4e..a1f0c34f6e 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/contributor/ArXivIdMetadataContributor.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/contributor/ArXivIdMetadataContributor.java @@ -22,6 +22,8 @@ import org.dspace.importer.external.metadatamapping.MetadataFieldMapping; import org.dspace.importer.external.metadatamapping.MetadatumDTO; import org.dspace.importer.external.metadatamapping.contributor.MetadataContributor; import org.jaxen.JaxenException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Required; /** @@ -34,6 +36,8 @@ import org.springframework.beans.factory.annotation.Required; public class ArXivIdMetadataContributor implements MetadataContributor { private MetadataFieldConfig field; + private static final Logger log = LoggerFactory.getLogger(ArXivIdMetadataContributor.class); + /** * Return prefixToNamespaceMapping * @@ -160,13 +164,13 @@ public class ArXivIdMetadataContributor implements MetadataContributor { private MetadataFieldConfig field; + private static final Logger log = LoggerFactory.getLogger(SimpleXpathMetadatumContributor.class); + /** * Return prefixToNamespaceMapping * @@ -157,12 +161,12 @@ public class SimpleXpathMetadatumContributor implements MetadataContributor Date: Tue, 28 Jul 2020 16:16:05 +0200 Subject: [PATCH 31/96] [Task 72215] removed the eperson flag from the REST metadata import script whilst leaving cli unaltered --- .../dspace/app/bulkedit/MetadataImport.java | 31 ++++------ .../app/bulkedit/MetadataImportCLI.java | 27 ++++++++ .../MetadataImportCliScriptConfiguration.java | 11 ++++ .../MetadataImportScriptConfiguration.java | 3 - .../main/java/org/dspace/eperson/EPerson.java | 2 +- .../app/bulkedit/MetadataImportTest.java | 32 ++++++++++ .../csv/CSVMetadataImportReferenceIT.java | 6 +- .../org/dspace/app/rest/csv/CsvImportIT.java | 61 ++++++++++++++++++- 8 files changed, 146 insertions(+), 27 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImport.java b/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImport.java index eb0a4e2935..5385882eee 100644 --- a/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImport.java +++ b/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImport.java @@ -182,24 +182,7 @@ public class MetadataImport extends DSpaceRunnable { + + @Override + public Options getOptions() { + Options options = super.getOptions(); + options.addOption("e", "email", true, "email address or user id of user (required if adding new items)"); + options.getOption("e").setType(String.class); + options.getOption("e").setRequired(true); + super.options = options; + return options; + } } diff --git a/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImportScriptConfiguration.java b/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImportScriptConfiguration.java index 9ea50b7de5..07e6a9aec9 100644 --- a/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImportScriptConfiguration.java +++ b/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImportScriptConfiguration.java @@ -57,9 +57,6 @@ public class MetadataImportScriptConfiguration extends options.addOption("f", "file", true, "source file"); options.getOption("f").setType(InputStream.class); options.getOption("f").setRequired(true); - options.addOption("e", "email", true, "email address or user id of user (required if adding new items)"); - options.getOption("e").setType(String.class); - options.getOption("e").setRequired(true); options.addOption("s", "silent", false, "silent operation - doesn't request confirmation of changes USE WITH CAUTION"); options.getOption("s").setType(boolean.class); diff --git a/dspace-api/src/main/java/org/dspace/eperson/EPerson.java b/dspace-api/src/main/java/org/dspace/eperson/EPerson.java index fc2950ee2b..3c48a5244a 100644 --- a/dspace-api/src/main/java/org/dspace/eperson/EPerson.java +++ b/dspace-api/src/main/java/org/dspace/eperson/EPerson.java @@ -141,7 +141,7 @@ public class EPerson extends DSpaceObject implements DSpaceObjectLegacySupport { return false; } final EPerson other = (EPerson) obj; - if (this.getID() != other.getID()) { + if (!this.getID().equals(other.getID())) { return false; } if (!StringUtils.equals(this.getEmail(), other.getEmail())) { diff --git a/dspace-api/src/test/java/org/dspace/app/bulkedit/MetadataImportTest.java b/dspace-api/src/test/java/org/dspace/app/bulkedit/MetadataImportTest.java index c0eb2789bc..5b053c7812 100644 --- a/dspace-api/src/test/java/org/dspace/app/bulkedit/MetadataImportTest.java +++ b/dspace-api/src/test/java/org/dspace/app/bulkedit/MetadataImportTest.java @@ -7,10 +7,12 @@ */ package org.dspace.app.bulkedit; +import static junit.framework.TestCase.assertEquals; import static junit.framework.TestCase.assertTrue; import java.io.File; +import org.apache.commons.cli.ParseException; import org.apache.commons.lang3.StringUtils; import org.dspace.AbstractIntegrationTest; import org.dspace.app.launcher.ScriptLauncher; @@ -22,9 +24,15 @@ import org.dspace.content.factory.ContentServiceFactory; import org.dspace.content.service.CollectionService; import org.dspace.content.service.CommunityService; import org.dspace.content.service.ItemService; +import org.dspace.scripts.DSpaceRunnable; +import org.dspace.scripts.configuration.ScriptConfiguration; +import org.dspace.scripts.factory.ScriptServiceFactory; +import org.dspace.scripts.service.ScriptService; import org.dspace.services.ConfigurationService; import org.dspace.services.factory.DSpaceServicesFactory; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; public class MetadataImportTest extends AbstractIntegrationTest { @@ -33,6 +41,9 @@ public class MetadataImportTest extends AbstractIntegrationTest { private CommunityService communityService = ContentServiceFactory.getInstance().getCommunityService(); private ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService(); + @Rule + public ExpectedException thrown = ExpectedException.none(); + @Test public void metadataImportTest() throws Exception { context.turnOffAuthorisationSystem(); @@ -50,6 +61,7 @@ public class MetadataImportTest extends AbstractIntegrationTest { StringUtils.equals( itemService.getMetadata(importedItem, "dc", "contributor", "author", Item.ANY).get(0).getValue(), "Donald, SmithImported")); + assertEquals(importedItem.getSubmitter(), eperson); context.turnOffAuthorisationSystem(); itemService.delete(context, itemService.find(context, importedItem.getID())); @@ -57,4 +69,24 @@ public class MetadataImportTest extends AbstractIntegrationTest { communityService.delete(context, communityService.find(context, community.getID())); context.restoreAuthSystemState(); } + + @Test(expected = ParseException.class) + public void metadataImportWithoutEPersonParameterTest() + throws IllegalAccessException, InstantiationException, ParseException { + String fileLocation = new File(testProps.get("test.importcsv").toString()).getAbsolutePath(); + String[] args = new String[] {"metadata-import", "-f", fileLocation, "-s"}; + TestDSpaceRunnableHandler testDSpaceRunnableHandler = new TestDSpaceRunnableHandler(); + + ScriptService scriptService = ScriptServiceFactory.getInstance().getScriptService(); + ScriptConfiguration scriptConfiguration = scriptService.getScriptConfiguration(args[0]); + + DSpaceRunnable script = null; + if (scriptConfiguration != null) { + script = scriptService.createDSpaceRunnableForScriptConfiguration(scriptConfiguration); + } + if (script != null) { + script.initialize(args, testDSpaceRunnableHandler, null); + script.run(); + } + } } diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CSVMetadataImportReferenceIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CSVMetadataImportReferenceIT.java index 53f3966c7c..ec870b0378 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CSVMetadataImportReferenceIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CSVMetadataImportReferenceIT.java @@ -479,11 +479,9 @@ public class CSVMetadataImportReferenceIT extends AbstractEntityIntegrationTest out.close(); try { if (validateOnly) { - return runDSpaceScript("metadata-import", "-f", csvFile.getAbsolutePath(), "-e", "admin@email.com", - "-s", "-v"); + return runDSpaceScript("metadata-import", "-f", csvFile.getAbsolutePath(), "-s", "-v"); } else { - return runDSpaceScript("metadata-import", "-f", csvFile.getAbsolutePath(), "-e", "admin@email.com", - "-s"); + return runDSpaceScript("metadata-import", "-f", csvFile.getAbsolutePath(), "-s"); } } finally { csvFile.delete(); diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CsvImportIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CsvImportIT.java index 5955edbdda..66900af744 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CsvImportIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CsvImportIT.java @@ -10,6 +10,7 @@ package org.dspace.app.rest.csv; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; @@ -24,6 +25,7 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import org.apache.commons.lang3.StringUtils; import org.dspace.app.rest.builder.CollectionBuilder; import org.dspace.app.rest.builder.CommunityBuilder; import org.dspace.app.rest.builder.ItemBuilder; @@ -233,11 +235,68 @@ public class CsvImportIT extends AbstractEntityIntegrationTest { out.close(); out = null; - runDSpaceScript("metadata-import", "-f", filename, "-e", "admin@email.com", "-s"); + runDSpaceScript("metadata-import", "-f", filename, "-s"); File file = new File(filename); if (file.exists()) { file.delete(); } } + + @Test + public void createRelationshipsWithCsvImportWithSpecifiedEPersonParameterTest() throws Exception { + context.turnOffAuthorisationSystem(); + + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Community child1 = CommunityBuilder.createSubCommunity(context, parentCommunity) + .withName("Sub Community") + .build(); + Collection col1 = CollectionBuilder.createCollection(context, child1).withName("Collection 1").build(); + Collection col2 = CollectionBuilder.createCollection(context, child1).withName("Collection 2").build(); + Collection col3 = CollectionBuilder.createCollection(context, child1).withName("OrgUnits").build(); + + Item article = ItemBuilder.createItem(context, col1) + .withTitle("Article") + .withIssueDate("2017-10-17") + .withRelationshipType("Publication") + .build(); + + String csvLineString = "+," + col1.getHandle() + ",TestItemB,Person," + article + .getID().toString(); + String[] csv = {"id,collection,dc.title,relationship.type,relation.isPublicationOfAuthor", csvLineString}; + + String filename = "test.csv"; + BufferedWriter out = new BufferedWriter( + new OutputStreamWriter( + new FileOutputStream(filename), "UTF-8")); + for (String csvLine : csv) { + out.write(csvLine + "\n"); + } + out.flush(); + out.close(); + out = null; + + runDSpaceScript("metadata-import", "-f", filename, "-e", "admin@email.com", "-s"); + + File file = new File(filename); + if (file.exists()) { + file.delete(); + } + + Iterator itemIteratorItem = itemService.findByMetadataField(context, "dc", "title", null, "TestItemB"); + Item item = itemIteratorItem.next(); + + List relationships = relationshipService.findByItem(context, item); + assertThat("Relationship list size is 1", relationships.size(), equalTo(1)); + getClient().perform(get("/api/core/items/" + item.getID())).andExpect(status().isOk()); + getClient().perform(get("/api/core/relationships/" + relationships.get(0).getID()).param("projection", "full")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.leftPlace", is(0))) + .andExpect(jsonPath("$.rightPlace", is(0))) + .andExpect(jsonPath("$", Matchers.is(RelationshipMatcher.matchRelationship(relationships.get(0))))); + + assertFalse(StringUtils.equalsIgnoreCase(item.getSubmitter().getEmail(), "admin@email.com")); + } } From 25bed852d39d0a95fdd90af77017346495ff521e Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Wed, 29 Jul 2020 01:19:20 +0200 Subject: [PATCH 32/96] Add comment, rewrite ArxivIdMetadataContributor --- .../ArXivIdMetadataContributor.java | 155 ++---------------- .../ArXivImportMetadataSourceServiceImpl.java | 39 ++++- 2 files changed, 50 insertions(+), 144 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/contributor/ArXivIdMetadataContributor.java b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/contributor/ArXivIdMetadataContributor.java index a1f0c34f6e..077fe85675 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/contributor/ArXivIdMetadataContributor.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/contributor/ArXivIdMetadataContributor.java @@ -8,23 +8,11 @@ package org.dspace.importer.external.arxiv.metadatamapping.contributor; import java.util.Collection; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import javax.annotation.Resource; -import org.apache.axiom.om.OMAttribute; import org.apache.axiom.om.OMElement; -import org.apache.axiom.om.OMText; -import org.apache.axiom.om.xpath.AXIOMXPath; -import org.dspace.importer.external.metadatamapping.MetadataFieldConfig; -import org.dspace.importer.external.metadatamapping.MetadataFieldMapping; import org.dspace.importer.external.metadatamapping.MetadatumDTO; import org.dspace.importer.external.metadatamapping.contributor.MetadataContributor; -import org.jaxen.JaxenException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Required; +import org.dspace.importer.external.metadatamapping.contributor.SimpleXpathMetadatumContributor; /** * Arxiv specific implementation of {@link MetadataContributor} @@ -33,109 +21,7 @@ import org.springframework.beans.factory.annotation.Required; * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it) * */ -public class ArXivIdMetadataContributor implements MetadataContributor { - private MetadataFieldConfig field; - - private static final Logger log = LoggerFactory.getLogger(ArXivIdMetadataContributor.class); - - /** - * Return prefixToNamespaceMapping - * - * @return a prefixToNamespaceMapping map - */ - public Map getPrefixToNamespaceMapping() { - return prefixToNamespaceMapping; - } - - private MetadataFieldMapping> metadataFieldMapping; - - /** - * Return metadataFieldMapping - * - * @return MetadataFieldMapping - */ - public MetadataFieldMapping> getMetadataFieldMapping() { - return metadataFieldMapping; - } - - /** - * Set the metadataFieldMapping of this ArXivIdMetadataContributor - * - * @param metadataFieldMapping the new mapping. - */ - public void setMetadataFieldMapping( - MetadataFieldMapping> metadataFieldMapping) { - this.metadataFieldMapping = metadataFieldMapping; - } - - /** - * Set the prefixToNamespaceMapping for this object, - * - * @param prefixToNamespaceMapping the new mapping. - */ - @Resource(name = "isiFullprefixMapping") - public void setPrefixToNamespaceMapping(Map prefixToNamespaceMapping) { - this.prefixToNamespaceMapping = prefixToNamespaceMapping; - } - - private Map prefixToNamespaceMapping; - - /** - * Initialize ArXivIdMetadataContributor with a query, prefixToNamespaceMapping and MetadataFieldConfig - * - * @param query query string - * @param prefixToNamespaceMapping metadata prefix to namespace mapping - * @param field - * MetadataFieldConfig - */ - public ArXivIdMetadataContributor(String query, Map prefixToNamespaceMapping, - MetadataFieldConfig field) { - this.query = query; - this.prefixToNamespaceMapping = prefixToNamespaceMapping; - this.field = field; - } - - /** - * Empty constructor for ArXivIdMetadataContributor - */ - public ArXivIdMetadataContributor() { - - } - - private String query; - - /** - * Return the MetadataFieldConfig used while retrieving MetadatumDTO - * - * @return MetadataFieldConfig - */ - public MetadataFieldConfig getField() { - return field; - } - - /** - * Setting the MetadataFieldConfig - * - * @param field MetadataFieldConfig used while retrieving MetadatumDTO - */ - @Required - public void setField(MetadataFieldConfig field) { - this.field = field; - } - - /** - * Return query used to create an xpathExpression on, this query is used to - * - * @return the query this instance is based on - */ - public String getQuery() { - return query; - } - - @Required - public void setQuery(String query) { - this.query = query; - } +public class ArXivIdMetadataContributor extends SimpleXpathMetadatumContributor { /** * Retrieve the metadata associated with the given object. @@ -147,35 +33,18 @@ public class ArXivIdMetadataContributor implements MetadataContributor contributeMetadata(OMElement t) { - List values = new LinkedList<>(); - try { - AXIOMXPath xpath = new AXIOMXPath(query); - for (String ns : prefixToNamespaceMapping.keySet()) { - xpath.addNamespace(prefixToNamespaceMapping.get(ns), ns); - } - List nodes = xpath.selectNodes(t); - for (Object el : nodes) { - if (el instanceof OMElement) { - values.add(metadataFieldMapping.toDCValue(field, ((OMElement) el).getText())); - } else if (el instanceof OMAttribute) { - values.add(metadataFieldMapping.toDCValue(field, ((OMAttribute) el).getAttributeValue())); - } else if (el instanceof String) { - values.add(metadataFieldMapping.toDCValue(field, (String) el)); - } else if (el instanceof OMText) { - values.add(metadataFieldMapping.toDCValue(field, ((OMText) el).getText())); - } else { - log.error("node of type: " + el.getClass()); - } - } - parseValue(values); - return values; - } catch (JaxenException e) { - log.error(query); - throw new RuntimeException(e); - } + Collection values = super.contributeMetadata(t); + parseValue(values); + return values; } - private void parseValue(List dtos) { + /** + * ArXiv doesn't return the item id. We have to get this from the path parameter + * + * @param dtos Metadata which contains the items uri + * @return the items ids + */ + private void parseValue(Collection dtos) { if (dtos != null) { for (MetadatumDTO dto : dtos) { if (dto != null && dto.getValue() != null && dto.getValue().contains("/")) { diff --git a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java index 863a8144b4..4320856842 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java @@ -164,6 +164,13 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata return retry(new FindMatchingRecordCallable(query)); } + /** + * This class is a Callable implementation to count the number of entries for an ArXiv + * query. + * + * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it) + * + */ private class CountByQueryCallable implements Callable { private Query query; @@ -207,7 +214,14 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata } } - + /** + * This class is a Callable implementation to get ArXiv entries based on + * query object. + * + * @see org.dspace.importer.external.datamodel.Query + * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it) + * + */ private class SearchByQueryCallable implements Callable> { private Query query; @@ -239,6 +253,9 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata } Invocation.Builder invocationBuilder = local.request(MediaType.TEXT_PLAIN_TYPE); Response response = invocationBuilder.get(); + if (response.getStatus() == 400) { + throw new IllegalArgumentException("Invalid ArXiv ID"); + } String responseString = response.readEntity(String.class); List omElements = splitToRecords(responseString); for (OMElement record : omElements) { @@ -248,6 +265,12 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata } } + /** + * This class is a Callable implementation to get ArXiv entry using ArXiv ID + * + * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it) + * + */ private class SearchByIdCallable implements Callable> { private Query query; @@ -275,6 +298,9 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata WebTarget local = webTarget.queryParam("id_list", arxivid); Invocation.Builder invocationBuilder = local.request(MediaType.TEXT_PLAIN_TYPE); Response response = invocationBuilder.get(); + if (response.getStatus() == 400) { + throw new IllegalArgumentException("Invalid ArXiv ID"); + } String responseString = response.readEntity(String.class); List omElements = splitToRecords(responseString); for (OMElement record : omElements) { @@ -284,6 +310,14 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata } } + /** + * This class is a Callable implementation to search ArXiv entries + * using author and title. + * + * @see org.dspace.importer.external.datamodel.Query + * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it) + * + */ private class FindMatchingRecordCallable implements Callable> { private Query query; @@ -299,6 +333,9 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata WebTarget local = webTarget.queryParam("search_query", queryString); Invocation.Builder invocationBuilder = local.request(MediaType.TEXT_PLAIN_TYPE); Response response = invocationBuilder.get(); + if (response.getStatus() == 400) { + throw new IllegalArgumentException("Invalid ArXiv ID"); + } String responseString = response.readEntity(String.class); List omElements = splitToRecords(responseString); for (OMElement record : omElements) { From afd6436c5c3b65697692c741244b258f9a94546b Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Wed, 29 Jul 2020 01:44:25 +0200 Subject: [PATCH 33/96] Clean code --- .../service/ArXivImportMetadataSourceServiceImpl.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java index 4320856842..869aaecd36 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java @@ -253,9 +253,6 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata } Invocation.Builder invocationBuilder = local.request(MediaType.TEXT_PLAIN_TYPE); Response response = invocationBuilder.get(); - if (response.getStatus() == 400) { - throw new IllegalArgumentException("Invalid ArXiv ID"); - } String responseString = response.readEntity(String.class); List omElements = splitToRecords(responseString); for (OMElement record : omElements) { @@ -298,9 +295,6 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata WebTarget local = webTarget.queryParam("id_list", arxivid); Invocation.Builder invocationBuilder = local.request(MediaType.TEXT_PLAIN_TYPE); Response response = invocationBuilder.get(); - if (response.getStatus() == 400) { - throw new IllegalArgumentException("Invalid ArXiv ID"); - } String responseString = response.readEntity(String.class); List omElements = splitToRecords(responseString); for (OMElement record : omElements) { @@ -333,9 +327,6 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata WebTarget local = webTarget.queryParam("search_query", queryString); Invocation.Builder invocationBuilder = local.request(MediaType.TEXT_PLAIN_TYPE); Response response = invocationBuilder.get(); - if (response.getStatus() == 400) { - throw new IllegalArgumentException("Invalid ArXiv ID"); - } String responseString = response.readEntity(String.class); List omElements = splitToRecords(responseString); for (OMElement record : omElements) { From 8df7e572b53550103c2969f6e4f64d83dfd85338 Mon Sep 17 00:00:00 2001 From: Raf Ponsaerts Date: Wed, 29 Jul 2020 09:05:13 +0200 Subject: [PATCH 34/96] [Task 72215] small bugfix in MetadataImport scripts --- .../main/java/org/dspace/app/bulkedit/MetadataImport.java | 3 --- .../java/org/dspace/app/bulkedit/MetadataImportCLI.java | 8 ++++++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImport.java b/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImport.java index 5385882eee..67086c1536 100644 --- a/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImport.java +++ b/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImport.java @@ -307,9 +307,6 @@ public class MetadataImport extends DSpaceRunnable Date: Wed, 29 Jul 2020 17:56:48 +0200 Subject: [PATCH 35/96] 72351: sitemaps path to config + in tests & script extension to delete sitemaps dir > called at end of tests & test for malicious file traversal --- .../dspace/app/sitemap/GenerateSitemaps.java | 33 +++++++++++--- .../app/rest/SitemapRestController.java | 2 +- .../app/rest/SitemapRestControllerIT.java | 44 +++++++++++++++---- dspace/config/dspace.cfg | 3 ++ 4 files changed, 65 insertions(+), 17 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java b/dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java index 907d34a47b..1729ef92f4 100644 --- a/dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java +++ b/dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java @@ -27,6 +27,7 @@ import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.cli.PosixParser; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; @@ -61,8 +62,6 @@ public class GenerateSitemaps { private static final ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService(); - public static final String SITEMAPS_ENDPOINT = "/sitemaps"; - /** * Default constructor */ @@ -86,6 +85,9 @@ public class GenerateSitemaps { options .addOption("p", "ping", true, "ping specified search engine URL"); + options + .addOption("d", "delete", false, + "delete sitemaps dir and its contents"); CommandLine line = null; @@ -107,10 +109,9 @@ public class GenerateSitemaps { } /* - * Sanity check -- if no sitemap generation or pinging to do, print - * usage + * Sanity check -- if no sitemap generation or pinging to do, or deletion, print usage */ - if (line.getArgs().length != 0 || line.hasOption('b') + if (line.getArgs().length != 0 || line.hasOption('d') || line.hasOption('b') && line.hasOption('s') && !line.hasOption('g') && !line.hasOption('m') && !line.hasOption('y') && !line.hasOption('p')) { @@ -125,6 +126,10 @@ public class GenerateSitemaps { generateSitemaps(!line.hasOption('b'), !line.hasOption('s')); } + if (line.hasOption('d')) { + deleteSitemaps(); + } + if (line.hasOption('a')) { pingConfiguredSearchEngines(); } @@ -152,6 +157,19 @@ public class GenerateSitemaps { generateSitemaps(true, true); } + /** + * Delete the sitemaps directory and its contents if it exists + * @throws IOException if IO error occurs + */ + public static void deleteSitemaps() throws IOException { + File outputDir = new File(configurationService.getProperty("sitemap.dir")); + if (!outputDir.exists() && !outputDir.isDirectory()) { + log.error("Unable to delete sitemaps directory, doesn't exist or isn't a directort"); + } else { + FileUtils.deleteDirectory(outputDir); + } + } + /** * Generate sitemap.org protocol and/or basic HTML sitemaps. * @@ -163,8 +181,9 @@ public class GenerateSitemaps { * if IO error occurs. */ public static void generateSitemaps(boolean makeHTMLMap, boolean makeSitemapOrg) throws SQLException, IOException { - String sitemapStem = configurationService.getProperty("dspace.server.url") - + SITEMAPS_ENDPOINT + "/sitemap"; + String sitemapsEndpoint = configurationService.getProperty("sitemap.path", "sitemaps"); + String sitemapStem = configurationService.getProperty("dspace.server.url") + "/" + + sitemapsEndpoint + "/sitemap"; String uiURLStem = configurationService.getProperty("dspace.ui.url"); File outputDir = new File(configurationService.getProperty("sitemap.dir")); diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/SitemapRestController.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/SitemapRestController.java index f9e4281f51..4eef1ba34b 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/SitemapRestController.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/SitemapRestController.java @@ -43,7 +43,7 @@ import org.springframework.web.bind.annotation.RequestMapping; * @author Maria Verdonck (Atmire) on 08/07/2020 */ @Controller -@RequestMapping("/sitemaps") +@RequestMapping("/${sitemap.path:sitemaps}") public class SitemapRestController { private static final Logger log = org.apache.logging.log4j.LogManager.getLogger(SitemapRestController.class); diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/SitemapRestControllerIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/SitemapRestControllerIT.java index 032de65ac7..b428435ede 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/SitemapRestControllerIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/SitemapRestControllerIT.java @@ -15,11 +15,14 @@ import static org.springframework.test.web.servlet.request.MockMvcRequestBuilder import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import javax.servlet.ServletException; + import org.dspace.app.rest.test.AbstractControllerIntegrationTest; import org.dspace.content.Collection; import org.dspace.content.Community; import org.dspace.content.Item; import org.dspace.services.ConfigurationService; +import org.junit.After; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -35,7 +38,7 @@ public class SitemapRestControllerIT extends AbstractControllerIntegrationTest { @Autowired ConfigurationService configurationService; - private final static String SITEMAPS_ENDPOINT = "/sitemaps"; + private final static String SITEMAPS_ENDPOINT = "sitemaps"; private Item item1; private Item item2; @@ -43,9 +46,10 @@ public class SitemapRestControllerIT extends AbstractControllerIntegrationTest { @Before @Override public void setUp() throws Exception { - super.setUp(); + configurationService.setProperty("sitemap.path", SITEMAPS_ENDPOINT); + context.turnOffAuthorisationSystem(); Community community = createCommunity(context).build(); @@ -64,20 +68,42 @@ public class SitemapRestControllerIT extends AbstractControllerIntegrationTest { context.restoreAuthSystemState(); } + @After + public void destroy() throws Exception { + // delete sitemaps generated by tests in before + runDSpaceScript("generate-sitemaps", "-d"); + + super.destroy(); + } + @Test public void testSitemap_notValidSiteMapFile() throws Exception { //** WHEN ** //We attempt to retrieve a non valid sitemap file - getClient().perform(get(SITEMAPS_ENDPOINT + "/notValidSiteMapFile")) + getClient().perform(get("/" + SITEMAPS_ENDPOINT + "/notValidSiteMapFile")) //** THEN ** .andExpect(status().isNotFound()); } + @Test(expected = ServletException.class) + public void testSitemap_fileSystemTraversal_dspaceCfg() throws Exception { + //** WHEN ** + //We attempt to use endpoint for malicious file system traversal + getClient().perform(get("/" + SITEMAPS_ENDPOINT + "/%2e%2e/config/dspace.cfg")); + } + + @Test(expected = ServletException.class) + public void testSitemap_fileSystemTraversal_dspaceCfg2() throws Exception { + //** WHEN ** + //We attempt to use endpoint for malicious file system traversal + getClient().perform(get("/" + SITEMAPS_ENDPOINT + "/%2e%2e%2fconfig%2fdspace.cfg")); + } + @Test public void testSitemap_sitemapIndexHtml() throws Exception { //** WHEN ** //We retrieve sitemap_index.html - MvcResult result = getClient().perform(get(SITEMAPS_ENDPOINT + "/sitemap_index.html")) + MvcResult result = getClient().perform(get("/" + SITEMAPS_ENDPOINT + "/sitemap_index.html")) //** THEN ** .andExpect(status().isOk()) //We expect the content type to match @@ -86,14 +112,14 @@ public class SitemapRestControllerIT extends AbstractControllerIntegrationTest { String response = result.getResponse().getContentAsString(); // contains a link to /sitemaps/sitemap0.html - assertTrue(response.contains(SITEMAPS_ENDPOINT + "/sitemap0.html")); + assertTrue(response.contains("/" + SITEMAPS_ENDPOINT + "/sitemap0.html")); } @Test public void testSitemap_sitemap0Html() throws Exception { //** WHEN ** //We retrieve sitemap0.html - MvcResult result = getClient().perform(get(SITEMAPS_ENDPOINT + "/sitemap0.html")) + MvcResult result = getClient().perform(get("/" + SITEMAPS_ENDPOINT + "/sitemap0.html")) //** THEN ** .andExpect(status().isOk()) //We expect the content type to match @@ -110,7 +136,7 @@ public class SitemapRestControllerIT extends AbstractControllerIntegrationTest { public void testSitemap_sitemapIndexXml() throws Exception { //** WHEN ** //We retrieve sitemap_index.xml - MvcResult result = getClient().perform(get(SITEMAPS_ENDPOINT + "/sitemap_index.xml")) + MvcResult result = getClient().perform(get("/" + SITEMAPS_ENDPOINT + "/sitemap_index.xml")) //** THEN ** .andExpect(status().isOk()) //We expect the content type to match @@ -119,14 +145,14 @@ public class SitemapRestControllerIT extends AbstractControllerIntegrationTest { String response = result.getResponse().getContentAsString(); // contains a link to /sitemaps/sitemap0.html - assertTrue(response.contains(SITEMAPS_ENDPOINT + "/sitemap0.xml")); + assertTrue(response.contains("/" + SITEMAPS_ENDPOINT + "/sitemap0.xml")); } @Test public void testSitemap_sitemap0Xml() throws Exception { //** WHEN ** //We retrieve sitemap0.html - MvcResult result = getClient().perform(get(SITEMAPS_ENDPOINT + "/sitemap0.xml")) + MvcResult result = getClient().perform(get("/" + SITEMAPS_ENDPOINT + "/sitemap0.xml")) //** THEN ** .andExpect(status().isOk()) //We expect the content type to match diff --git a/dspace/config/dspace.cfg b/dspace/config/dspace.cfg index 1bbd895131..6550b2e3d3 100644 --- a/dspace/config/dspace.cfg +++ b/dspace/config/dspace.cfg @@ -1429,6 +1429,9 @@ webui.content_disposition_threshold = 8388608 # the directory where the generated sitemaps are stored sitemap.dir = ${dspace.dir}/sitemaps +# sitemaps endpoint path used in SitemapRestController and in the url paths when sitemaps are generated +sitemap.path = sitemaps + # # Comma-separated list of search engine URLs to 'ping' when a new Sitemap has # been created. Include everything except the Sitemap URL itself (which will From 95b5cabae25e75379c1a5090a3a4fdf0d427d0cf Mon Sep 17 00:00:00 2001 From: Ben Bosman Date: Thu, 30 Jul 2020 09:06:53 +0200 Subject: [PATCH 36/96] trailing whitespace --- .../java/org/dspace/app/rest/builder/WorkspaceItemBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/builder/WorkspaceItemBuilder.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/builder/WorkspaceItemBuilder.java index 2c515d2b2c..eb4a096be1 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/builder/WorkspaceItemBuilder.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/builder/WorkspaceItemBuilder.java @@ -166,7 +166,7 @@ public class WorkspaceItemBuilder extends AbstractBuilder Date: Thu, 30 Jul 2020 15:58:54 +0200 Subject: [PATCH 37/96] [Task 72215] made sure that the script rest calls in tests are single threaded and fixed CsvImportIT tests --- .../impl/RestDSpaceRunnableHandler.java | 7 +- .../config/spring/rest/scripts.xml | 15 +++ .../org/dspace/app/rest/csv/CsvImportIT.java | 124 ++++++++++++------ 3 files changed, 103 insertions(+), 43 deletions(-) create mode 100644 dspace-server-webapp/src/test/data/dspaceFolder/config/spring/rest/scripts.xml diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java index f2080dcd84..e5ef6f43a2 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java @@ -33,6 +33,7 @@ import org.dspace.scripts.factory.ScriptServiceFactory; import org.dspace.scripts.handler.DSpaceRunnableHandler; import org.dspace.scripts.service.ProcessService; import org.dspace.utils.DSpace; +import org.springframework.core.task.TaskExecutor; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; /** @@ -231,9 +232,9 @@ public class RestDSpaceRunnableHandler implements DSpaceRunnableHandler { * @param script The script to be ran */ public void schedule(DSpaceRunnable script) { - ThreadPoolTaskExecutor taskExecutor = new DSpace().getServiceManager() - .getServiceByName("dspaceRunnableThreadExecutor", - ThreadPoolTaskExecutor.class); +// script.run(); + TaskExecutor taskExecutor = new DSpace().getServiceManager() + .getServiceByName("dspaceRunnableThreadExecutor", TaskExecutor.class); Context context = new Context(); try { Process process = processService.find(context, processId); diff --git a/dspace-server-webapp/src/test/data/dspaceFolder/config/spring/rest/scripts.xml b/dspace-server-webapp/src/test/data/dspaceFolder/config/spring/rest/scripts.xml new file mode 100644 index 0000000000..7eb5ac7270 --- /dev/null +++ b/dspace-server-webapp/src/test/data/dspaceFolder/config/spring/rest/scripts.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CsvImportIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CsvImportIT.java index 66900af744..98a673e28f 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CsvImportIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CsvImportIT.java @@ -7,29 +7,45 @@ */ package org.dspace.app.rest.csv; +import static com.jayway.jsonpath.JsonPath.read; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.fileUpload; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import java.io.BufferedWriter; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileOutputStream; +import java.io.InputStream; import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import com.google.gson.Gson; import org.apache.commons.lang3.StringUtils; import org.dspace.app.rest.builder.CollectionBuilder; import org.dspace.app.rest.builder.CommunityBuilder; import org.dspace.app.rest.builder.ItemBuilder; +import org.dspace.app.rest.builder.ProcessBuilder; +import org.dspace.app.rest.converter.DSpaceRunnableParameterConverter; +import org.dspace.app.rest.matcher.ProcessMatcher; import org.dspace.app.rest.matcher.RelationshipMatcher; +import org.dspace.app.rest.model.ParameterValueRest; +import org.dspace.app.rest.projection.Projection; import org.dspace.app.rest.test.AbstractEntityIntegrationTest; import org.dspace.content.Collection; import org.dspace.content.Community; @@ -39,9 +55,12 @@ import org.dspace.content.service.EntityTypeService; import org.dspace.content.service.ItemService; import org.dspace.content.service.RelationshipService; import org.dspace.content.service.RelationshipTypeService; +import org.dspace.scripts.DSpaceCommandLineParameter; import org.hamcrest.Matchers; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.MediaType; +import org.springframework.mock.web.MockMultipartFile; public class CsvImportIT extends AbstractEntityIntegrationTest { @@ -57,6 +76,9 @@ public class CsvImportIT extends AbstractEntityIntegrationTest { @Autowired private ItemService itemService; + @Autowired + private DSpaceRunnableParameterConverter dSpaceRunnableParameterConverter; + @Test public void createRelationshipsWithCsvImportTest() throws Exception { context.turnOffAuthorisationSystem(); @@ -121,6 +143,8 @@ public class CsvImportIT extends AbstractEntityIntegrationTest { assertArticleRelationships(article, itemB, itemC, itemF); + + } private void assertItemERelationships(Item itemB, Item itemE, Item itemF) throws SQLException { @@ -224,22 +248,37 @@ public class CsvImportIT extends AbstractEntityIntegrationTest { } private void performImportScript(String[] csv) throws Exception { - String filename = "test.csv"; - BufferedWriter out = new BufferedWriter( - new OutputStreamWriter( - new FileOutputStream(filename), "UTF-8")); - for (String csvLine : csv) { - out.write(csvLine + "\n"); - } - out.flush(); - out.close(); - out = null; + InputStream inputStream = new ByteArrayInputStream(String.join(System.lineSeparator(), Arrays.asList(csv)).getBytes( + StandardCharsets.UTF_8)); - runDSpaceScript("metadata-import", "-f", filename, "-s"); + MockMultipartFile bitstreamFile = new MockMultipartFile("file", + "test.csv", MediaType.TEXT_PLAIN_VALUE, + inputStream); - File file = new File(filename); - if (file.exists()) { - file.delete(); + AtomicReference idRef = new AtomicReference<>(); + + LinkedList parameters = new LinkedList<>(); + parameters.add(new DSpaceCommandLineParameter("-f", "test.csv")); + parameters.add(new DSpaceCommandLineParameter("-s", "")); + + List list = parameters.stream() + .map(dSpaceCommandLineParameter -> dSpaceRunnableParameterConverter + .convert(dSpaceCommandLineParameter, Projection.DEFAULT)) + .collect(Collectors.toList()); + + try { + String token = getAuthToken(admin.getEmail(), password); + + getClient(token) + .perform(fileUpload("/api/system/scripts/metadata-import/processes").file(bitstreamFile) + .param("properties", + new Gson().toJson(list))) + .andExpect(status().isAccepted()) + .andDo(result -> idRef + .set(read(result.getResponse().getContentAsString(), "$.processId"))); + String t = ""; + } finally { + ProcessBuilder.deleteProcess(idRef.get()); } } @@ -267,36 +306,41 @@ public class CsvImportIT extends AbstractEntityIntegrationTest { .getID().toString(); String[] csv = {"id,collection,dc.title,relationship.type,relation.isPublicationOfAuthor", csvLineString}; - String filename = "test.csv"; - BufferedWriter out = new BufferedWriter( - new OutputStreamWriter( - new FileOutputStream(filename), "UTF-8")); - for (String csvLine : csv) { - out.write(csvLine + "\n"); - } - out.flush(); - out.close(); - out = null; + InputStream inputStream = new ByteArrayInputStream(String.join(System.lineSeparator(), Arrays.asList(csv)).getBytes( + StandardCharsets.UTF_8)); - runDSpaceScript("metadata-import", "-f", filename, "-e", "admin@email.com", "-s"); + MockMultipartFile bitstreamFile = new MockMultipartFile("file", + "test.csv", MediaType.TEXT_PLAIN_VALUE, + inputStream); - File file = new File(filename); - if (file.exists()) { - file.delete(); + AtomicReference idRef = new AtomicReference<>(); + + LinkedList parameters = new LinkedList<>(); + parameters.add(new DSpaceCommandLineParameter("-f", "test.csv")); + parameters.add(new DSpaceCommandLineParameter("-s", "")); + parameters.add(new DSpaceCommandLineParameter("-e", "dspace@dspace.com")); + + List list = parameters.stream() + .map(dSpaceCommandLineParameter -> dSpaceRunnableParameterConverter + .convert(dSpaceCommandLineParameter, Projection.DEFAULT)) + .collect(Collectors.toList()); + + try { + String token = getAuthToken(admin.getEmail(), password); + + getClient(token) + .perform(fileUpload("/api/system/scripts/metadata-import/processes").file(bitstreamFile) + .param("properties", + new Gson().toJson(list))) + .andExpect(status().isAccepted()) + .andDo(result -> idRef + .set(read(result.getResponse().getContentAsString(), "$.processId"))); + String t = ""; + } finally { + ProcessBuilder.deleteProcess(idRef.get()); } Iterator itemIteratorItem = itemService.findByMetadataField(context, "dc", "title", null, "TestItemB"); - Item item = itemIteratorItem.next(); - - List relationships = relationshipService.findByItem(context, item); - assertThat("Relationship list size is 1", relationships.size(), equalTo(1)); - getClient().perform(get("/api/core/items/" + item.getID())).andExpect(status().isOk()); - getClient().perform(get("/api/core/relationships/" + relationships.get(0).getID()).param("projection", "full")) - .andExpect(status().isOk()) - .andExpect(jsonPath("$.leftPlace", is(0))) - .andExpect(jsonPath("$.rightPlace", is(0))) - .andExpect(jsonPath("$", Matchers.is(RelationshipMatcher.matchRelationship(relationships.get(0))))); - - assertFalse(StringUtils.equalsIgnoreCase(item.getSubmitter().getEmail(), "admin@email.com")); + assertFalse(itemIteratorItem.hasNext()); } } From fd63d6216e88ee3299f5872ff489a672510ee39c Mon Sep 17 00:00:00 2001 From: Raf Ponsaerts Date: Fri, 31 Jul 2020 11:39:22 +0200 Subject: [PATCH 38/96] [Task 72215] moved CSVMetadataImportreferenceIT to the dspace-api module and fixed tests --- .../csv/CSVMetadataImportReferenceIT.java | 314 ++++++++++++------ .../impl/RestDSpaceRunnableHandler.java | 1 - .../org/dspace/app/rest/csv/CsvImportIT.java | 15 +- 3 files changed, 223 insertions(+), 107 deletions(-) rename {dspace-server-webapp/src/test/java/org/dspace/app/rest => dspace-api/src/test/java/org/dspace/app}/csv/CSVMetadataImportReferenceIT.java (58%) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CSVMetadataImportReferenceIT.java b/dspace-api/src/test/java/org/dspace/app/csv/CSVMetadataImportReferenceIT.java similarity index 58% rename from dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CSVMetadataImportReferenceIT.java rename to dspace-api/src/test/java/org/dspace/app/csv/CSVMetadataImportReferenceIT.java index ec870b0378..5ecbd48ba5 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CSVMetadataImportReferenceIT.java +++ b/dspace-api/src/test/java/org/dspace/app/csv/CSVMetadataImportReferenceIT.java @@ -5,7 +5,7 @@ * * http://www.dspace.org/license/ */ -package org.dspace.app.rest.csv; +package org.dspace.app.csv; import static junit.framework.TestCase.assertEquals; @@ -19,52 +19,130 @@ import java.util.Iterator; import java.util.List; import java.util.UUID; +import org.dspace.AbstractIntegrationTest; import org.dspace.app.bulkedit.MetadataImportException; import org.dspace.app.bulkedit.MetadataImportInvalidHeadingException; -import org.dspace.app.rest.builder.CollectionBuilder; -import org.dspace.app.rest.builder.CommunityBuilder; -import org.dspace.app.rest.builder.ItemBuilder; -import org.dspace.app.rest.test.AbstractEntityIntegrationTest; +import org.dspace.app.scripts.handler.impl.TestDSpaceRunnableHandler; +import org.dspace.authorize.AuthorizeException; import org.dspace.content.Collection; +import org.dspace.content.Community; +import org.dspace.content.EntityType; import org.dspace.content.Item; import org.dspace.content.MetadataField; import org.dspace.content.MetadataValue; import org.dspace.content.Relationship; +import org.dspace.content.WorkspaceItem; import org.dspace.content.factory.ContentServiceFactory; +import org.dspace.content.service.CollectionService; +import org.dspace.content.service.CommunityService; +import org.dspace.content.service.EntityTypeService; +import org.dspace.content.service.InstallItemService; import org.dspace.content.service.ItemService; import org.dspace.content.service.MetadataFieldService; import org.dspace.content.service.MetadataValueService; import org.dspace.content.service.RelationshipService; +import org.dspace.content.service.RelationshipTypeService; +import org.dspace.content.service.WorkspaceItemService; +import org.dspace.scripts.DSpaceRunnable; +import org.dspace.scripts.configuration.ScriptConfiguration; +import org.dspace.scripts.factory.ScriptServiceFactory; +import org.dspace.scripts.service.ScriptService; +import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; /** * Created by: Andrew Wood * Date: 26 Jul 2019 */ -public class CSVMetadataImportReferenceIT extends AbstractEntityIntegrationTest { +public class CSVMetadataImportReferenceIT extends AbstractIntegrationTest { //Common collection to utilize for test private Collection col1; - @Autowired - private RelationshipService relationshipService; + private RelationshipService relationshipService = ContentServiceFactory.getInstance().getRelationshipService(); + private ItemService itemService = ContentServiceFactory.getInstance().getItemService(); + private CollectionService collectionService = ContentServiceFactory.getInstance().getCollectionService(); + private CommunityService communityService = ContentServiceFactory.getInstance().getCommunityService(); + private WorkspaceItemService workspaceItemService = ContentServiceFactory.getInstance().getWorkspaceItemService(); + private InstallItemService installItemService = ContentServiceFactory.getInstance().getInstallItemService(); + private EntityTypeService entityTypeService = ContentServiceFactory.getInstance().getEntityTypeService(); + private RelationshipTypeService relationshipTypeService = ContentServiceFactory.getInstance() + .getRelationshipTypeService(); - @Autowired - private ItemService itemService; + + Community parentCommunity; /** * Setup testing enviorment */ @Before - public void setup() { + public void setup() throws SQLException, AuthorizeException { context.turnOffAuthorisationSystem(); - parentCommunity = CommunityBuilder.createCommunity(context) - .withName("Parent Community") - .build(); - col1 = CollectionBuilder.createCollection(context, parentCommunity).withName("Collection 1").build(); + parentCommunity = communityService.create(null, context); + communityService.addMetadata(context, parentCommunity, "dc", "title", null, null, "Parent Community"); + + col1 = collectionService.create(context, parentCommunity); + collectionService.addMetadata(context, col1, "dc", "title", null, null, "Collection 1"); + + if (entityTypeService.findAll(context).size() > 0) { + //Don't initialize the setup more than once + return; + } + + context.turnOffAuthorisationSystem(); + + EntityType publicationEntityType = entityTypeService.create(context, "Publication"); + EntityType personEntityType = entityTypeService.create(context, "Person"); + EntityType orgUnitType = entityTypeService.create(context, "OrgUnit"); + EntityType projectType = entityTypeService.create(context, "Project"); + relationshipTypeService + .create(context, publicationEntityType, personEntityType, "isAuthorOfPublication", "isPublicationOfAuthor", + null, null, null, null); + relationshipTypeService + .create(context, publicationEntityType, projectType, "isProjectOfPublication", "isPublicationOfProject", 0, + null, 0, null, false, true); + context.restoreAuthSystemState(); + + context.restoreAuthSystemState(); + } + + @After + @Override + public void destroy() { + context.turnOffAuthorisationSystem(); + try { + List relationships = relationshipService.findAll(context); + for (Relationship relationship : relationships) { + relationshipService.delete(context, relationship); + } + Iterator itemIterator = itemService.findAll(context); + while (itemIterator.hasNext()) { + Item item = itemIterator.next(); + itemService.delete(context, item); + } + List collections = collectionService.findAll(context); + for (Collection collection : collections) { + collectionService.delete(context, collection); + } + List communities = communityService.findAll(context); + for (Community community : communities) { + communityService.delete(context, community); + } + context.commit(); + } catch (Exception e) { + String t = ""; + } + + context.restoreAuthSystemState(); + col1 = null; + parentCommunity = null; + try { + super.destroy(); + } catch (Exception e) { + String t = ""; + } } /** @@ -102,8 +180,8 @@ public class CSVMetadataImportReferenceIT extends AbstractEntityIntegrationTest @Test public void testSingleMdRef() throws Exception { String[] csv = {"id,relationship.type,relation.isAuthorOfPublication,collection,dc.identifier.other", - "+,Person,," + col1.getHandle() + ",0", - "+,Publication,dc.identifier.other:0," + col1.getHandle() + ",1"}; + "+,Person,," + col1.getHandle() + ",0", + "+,Publication,dc.identifier.other:0," + col1.getHandle() + ",1"}; Item[] items = runImport(csv); assertRelationship(items[1], items[0], 1, "left", 0); } @@ -119,7 +197,7 @@ public class CSVMetadataImportReferenceIT extends AbstractEntityIntegrationTest performImportScript(csvLines, false); Item[] items = new Item[csvLines.length - 1]; for (int i = 0; i < items.length; i++) { - items[i] = itemService.findByIdOrLegacyId(context, getUUIDByIdentifierOther("" + i).toString()); + items[i] = itemService.findByIdOrLegacyId(context, getUUIDByIdentifierOther("" + i).toString()); } return items; } @@ -132,8 +210,8 @@ public class CSVMetadataImportReferenceIT extends AbstractEntityIntegrationTest public void testSingleRowNameRef() throws Exception { String[] csv = {"id,dc.title,relationship.type,relation.isAuthorOfPublication,collection,rowName," + "dc.identifier.other", - "+,Test Item 1,Person,," + col1.getHandle() + ",idVal,0", - "+,Test Item 2,Publication,rowName:idVal," + col1.getHandle() + ",anything,1"}; + "+,Test Item 1,Person,," + col1.getHandle() + ",idVal,0", + "+,Test Item 2,Publication,rowName:idVal," + col1.getHandle() + ",anything,1"}; Item[] items = runImport(csv); assertRelationship(items[1], items[0], 1, "left", 0); } @@ -145,9 +223,9 @@ public class CSVMetadataImportReferenceIT extends AbstractEntityIntegrationTest @Test public void testMultiMdRef() throws Exception { String[] csv = {"id,relationship.type,relation.isAuthorOfPublication,collection,dc.identifier.other", - "+,Person,," + col1.getHandle() + ",0", - "+,Person,," + col1.getHandle() + ",1", - "+,Publication,dc.identifier.other:0||dc.identifier.other:1," + col1.getHandle() + ",2"}; + "+,Person,," + col1.getHandle() + ",0", + "+,Person,," + col1.getHandle() + ",1", + "+,Publication,dc.identifier.other:0||dc.identifier.other:1," + col1.getHandle() + ",2"}; Item[] items = runImport(csv); assertRelationship(items[2], items[0], 1, "left", 0); assertRelationship(items[2], items[1], 1, "left", 1); @@ -160,9 +238,9 @@ public class CSVMetadataImportReferenceIT extends AbstractEntityIntegrationTest @Test public void testMultiRowNameRef() throws Exception { String[] csv = {"id,relationship.type,relation.isAuthorOfPublication,collection,dc.identifier.other,rowName", - "+,Person,," + col1.getHandle() + ",0,val1", - "+,Person,," + col1.getHandle() + ",1,val2", - "+,Publication,rowName:val1||rowName:val2," + col1.getHandle() + ",2,val3"}; + "+,Person,," + col1.getHandle() + ",0,val1", + "+,Person,," + col1.getHandle() + ",1,val2", + "+,Publication,rowName:val1||rowName:val2," + col1.getHandle() + ",2,val3"}; Item[] items = runImport(csv); assertRelationship(items[2], items[0], 1, "left", 0); assertRelationship(items[2], items[1], 1, "left", 1); @@ -175,12 +253,13 @@ public class CSVMetadataImportReferenceIT extends AbstractEntityIntegrationTest @Test public void testSingleUUIDReference() throws Exception { context.turnOffAuthorisationSystem(); - Item person = ItemBuilder.createItem(context, col1) - .withRelationshipType("Person") - .build(); + WorkspaceItem workspaceItem = workspaceItemService.create(context, col1, false); + Item person = installItemService.installItem(context, workspaceItem); + itemService.addMetadata(context, person, "relationship", "type", null, null, "Person"); + itemService.update(context, person); context.restoreAuthSystemState(); String[] csv = {"id,relationship.type,relation.isAuthorOfPublication,collection,rowName,dc.identifier.other", - "+,Publication," + person.getID().toString() + "," + col1.getHandle() + ",anything,0"}; + "+,Publication," + person.getID().toString() + "," + col1.getHandle() + ",anything,0"}; Item[] items = runImport(csv); assertRelationship(items[0], person, 1, "left", 0); } @@ -192,12 +271,14 @@ public class CSVMetadataImportReferenceIT extends AbstractEntityIntegrationTest @Test public void testMultiUUIDReference() throws Exception { context.turnOffAuthorisationSystem(); - Item person = ItemBuilder.createItem(context, col1) - .withRelationshipType("Person") - .build(); - Item person2 = ItemBuilder.createItem(context, col1) - .withRelationshipType("Person") - .build(); + WorkspaceItem workspaceItem = workspaceItemService.create(context, col1, false); + Item person = installItemService.installItem(context, workspaceItem); + itemService.addMetadata(context, person, "relationship", "type", null, null, "Person"); + itemService.update(context, person); + WorkspaceItem workspaceItem2 = workspaceItemService.create(context, col1, false); + Item person2 = installItemService.installItem(context, workspaceItem2); + itemService.addMetadata(context, person2, "relationship", "type", null, null, "Person"); + itemService.update(context, person2); context.restoreAuthSystemState(); String[] csv = {"id,relationship.type,relation.isAuthorOfPublication,collection,rowName,dc.identifier.other", "+,Publication," + person.getID().toString() + "||" + person2.getID().toString() + "," + @@ -214,14 +295,16 @@ public class CSVMetadataImportReferenceIT extends AbstractEntityIntegrationTest @Test public void testMultiRefArchivedCsv() throws Exception { context.turnOffAuthorisationSystem(); - Item person = ItemBuilder.createItem(context, col1) - .withTitle("Person") - .withRelationshipType("Person") - .build(); + WorkspaceItem workspaceItem = workspaceItemService.create(context, col1, false); + Item person = installItemService.installItem(context, workspaceItem); + itemService.addMetadata(context, person, "relationship", "type", null, null, "Person"); + itemService.addMetadata(context, person, "dc", "title", null, null, "Person"); + itemService.update(context, person); + String[] csv = {"id,dc.title,relationship.type,relation.isAuthorOfPublication,collection,rowName," + "dc.identifier.other", - "+,Person2,Person,," + col1.getHandle() + ",idVal,0", - "+,Pub1,Publication,dc.title:Person||dc.title:Person2," + col1.getHandle() + ",anything,1"}; + "+,Person2,Person,," + col1.getHandle() + ",idVal,0", + "+,Pub1,Publication,dc.title:Person||dc.title:Person2," + col1.getHandle() + ",anything,1"}; context.restoreAuthSystemState(); Item[] items = runImport(csv); assertRelationship(items[1], person, 1, "left", 0); @@ -236,18 +319,21 @@ public class CSVMetadataImportReferenceIT extends AbstractEntityIntegrationTest @Test public void testMultiMixedRefArchivedCsv() throws Exception { context.turnOffAuthorisationSystem(); - Item person = ItemBuilder.createItem(context, col1) - .withTitle("Person") - .withRelationshipType("Person") - .build(); - Item person2 = ItemBuilder.createItem(context, col1) - .withTitle("Person2") - .withRelationshipType("Person") - .build(); + WorkspaceItem workspaceItem = workspaceItemService.create(context, col1, false); + Item person = installItemService.installItem(context, workspaceItem); + itemService.addMetadata(context, person, "relationship", "type", null, null, "Person"); + itemService.addMetadata(context, person, "dc", "title", null, null, "Person"); + itemService.update(context, person); + WorkspaceItem workspaceItem2 = workspaceItemService.create(context, col1, false); + Item person2 = installItemService.installItem(context, workspaceItem2); + itemService.addMetadata(context, person2, "relationship", "type", null, null, "Person"); + itemService.addMetadata(context, person2, "dc", "title", null, null, "Person2"); + itemService.update(context, person2); + context.restoreAuthSystemState(); String[] csv = {"id,dc.title,relationship.type,relation.isAuthorOfPublication,collection,rowName," + "dc.identifier.other", - "+,Person3,Person,," + col1.getHandle() + ",idVal,0", + "+,Person3,Person,," + col1.getHandle() + ",idVal,0", "+,Pub1,Publication," + person.getID() + "||dc.title:Person2||rowName:idVal," + col1.getHandle() + ",anything,1"}; Item[] items = runImport(csv); @@ -264,8 +350,8 @@ public class CSVMetadataImportReferenceIT extends AbstractEntityIntegrationTest public void testRefWithSpecialChar() throws Exception { String[] csv = {"id,dc.title,relationship.type,relation.isAuthorOfPublication,collection,rowName," + "dc.identifier.other", - "+,Person:,Person,," + col1.getHandle() + ",idVal,0", - "+,Pub1,Publication,dc.title:Person:," + col1.getHandle() + ",anything,1"}; + "+,Person:,Person,," + col1.getHandle() + ",idVal,0", + "+,Pub1,Publication,dc.title:Person:," + col1.getHandle() + ",anything,1"}; Item[] items = runImport(csv); assertRelationship(items[1], items[0], 1, "left", 0); } @@ -300,14 +386,17 @@ public class CSVMetadataImportReferenceIT extends AbstractEntityIntegrationTest @Test(expected = MetadataImportException.class) public void testNonUniqueMDRefInDb() throws Exception { context.turnOffAuthorisationSystem(); - ItemBuilder.createItem(context, col1) - .withRelationshipType("Person") - .withIdentifierOther("1") - .build(); - ItemBuilder.createItem(context, col1) - .withRelationshipType("Person") - .withIdentifierOther("1") - .build(); + WorkspaceItem workspaceItem = workspaceItemService.create(context, col1, false); + Item person = installItemService.installItem(context, workspaceItem); + itemService.addMetadata(context, person, "relationship", "type", null, null, "Person"); + itemService.addMetadata(context, person, "dc", "identifier", "other", null, "1"); + itemService.update(context, person); + WorkspaceItem workspaceItem2 = workspaceItemService.create(context, col1, false); + Item person2 = installItemService.installItem(context, workspaceItem2); + itemService.addMetadata(context, person2, "relationship", "type", null, null, "Person"); + itemService.addMetadata(context, person2, "dc", "identifier", "other", null, "1"); + itemService.update(context, person2); + context.restoreAuthSystemState(); String[] csv = {"id,relationship.type,relation.isAuthorOfPublication,collection,dc.identifier.other", "+,Publication,dc.identifier.other:1," + col1.getHandle() + ",2"}; @@ -320,10 +409,11 @@ public class CSVMetadataImportReferenceIT extends AbstractEntityIntegrationTest @Test(expected = MetadataImportException.class) public void testNonUniqueMDRefInBoth() throws Exception { context.turnOffAuthorisationSystem(); - ItemBuilder.createItem(context, col1) - .withRelationshipType("Person") - .withIdentifierOther("1") - .build(); + WorkspaceItem workspaceItem = workspaceItemService.create(context, col1, false); + Item person = installItemService.installItem(context, workspaceItem); + itemService.addMetadata(context, person, "relationship", "type", null, null, "Person"); + itemService.addMetadata(context, person, "dc", "identifier", "other", null, "1"); + itemService.update(context, person); context.restoreAuthSystemState(); String[] csv = {"id,relationship.type,relation.isAuthorOfPublication,collection,dc.identifier.other", "+,Person,," + col1.getHandle() + ",1", @@ -381,9 +471,10 @@ public class CSVMetadataImportReferenceIT extends AbstractEntityIntegrationTest @Test(expected = MetadataImportInvalidHeadingException.class) public void testInvalidRelationshipArchivedOrigin() throws Exception { context.turnOffAuthorisationSystem(); - Item testItem = ItemBuilder.createItem(context, col1) - .withRelationshipType("OrgUnit") - .build(); + WorkspaceItem workspaceItem = workspaceItemService.create(context, col1, false); + Item testItem = installItemService.installItem(context, workspaceItem); + itemService.addMetadata(context, testItem, "relationship", "type", null, null, "OrgUnit"); + itemService.update(context, testItem); context.restoreAuthSystemState(); String[] csv = {"id,relationship.type,relation.isAuthorOfPublication,collection,rowName", "+,Person,," + col1.getHandle() + ",1" + @@ -397,9 +488,10 @@ public class CSVMetadataImportReferenceIT extends AbstractEntityIntegrationTest @Test(expected = MetadataImportInvalidHeadingException.class) public void testInvalidRelationshipArchivedTarget() throws Exception { context.turnOffAuthorisationSystem(); - Item testItem = ItemBuilder.createItem(context, col1) - .withRelationshipType("OrgUnit") - .build(); + WorkspaceItem workspaceItem = workspaceItemService.create(context, col1, false); + Item testItem = installItemService.installItem(context, workspaceItem); + itemService.addMetadata(context, testItem, "relationship", "type", null, null, "OrgUnit"); + itemService.update(context, testItem); context.restoreAuthSystemState(); String[] csv = {"id,relationship.type,relation.isAuthorOfPublication,collection,rowName", testItem.getID().toString() + ",Person,," + col1.getHandle() + ",1" + @@ -413,26 +505,36 @@ public class CSVMetadataImportReferenceIT extends AbstractEntityIntegrationTest @Test public void testValidRelationshipNoDefinedTypesInCSV() throws Exception { context.turnOffAuthorisationSystem(); - Item testItemOne = ItemBuilder.createItem(context, col1) - .withRelationshipType("Person") - .withIdentifierOther("testItemOne") - .build(); - Item testItemTwo = ItemBuilder.createItem(context, col1) - .withRelationshipType("Publication") - .withIdentifierOther("testItemTwo") - .build(); - Item testItemThree = ItemBuilder.createItem(context, col1) - .withRelationshipType("Project") - .withIdentifierOther("testItemThree") - .build(); + + WorkspaceItem workspaceItem = workspaceItemService.create(context, col1, false); + Item testItem = installItemService.installItem(context, workspaceItem); + itemService.addMetadata(context, testItem, "relationship", "type", null, null, "Person"); + itemService.addMetadata(context, testItem, "dc", "identifier", "other", null, "testItemOne"); + itemService.update(context, testItem); + + + WorkspaceItem workspaceItem2 = workspaceItemService.create(context, col1, false); + Item testItem2 = installItemService.installItem(context, workspaceItem2); + itemService.addMetadata(context, testItem2, "relationship", "type", null, null, "Publication"); + itemService.addMetadata(context, testItem2, "dc", "identifier", "other", null, "testItemTwo"); + itemService.update(context, testItem2); + + + WorkspaceItem workspaceItem3 = workspaceItemService.create(context, col1, false); + Item testItem3 = installItemService.installItem(context, workspaceItem3); + itemService.addMetadata(context, testItem3, "relationship", "type", null, null, "Project"); + itemService.addMetadata(context, testItem3, "dc", "identifier", "other", null, "testItemThree"); + itemService.update(context, testItem3); + + context.restoreAuthSystemState(); String[] csv = {"id,relation.isAuthorOfPublication,relation.isPublicationOfProject,collection", - testItemOne.getID().toString() + ",,," + col1.getHandle(), - testItemTwo.getID().toString() + ",dc.identifier.other:testItemOne,," + col1.getHandle(), - testItemThree.getID().toString() + ",,dc.identifier.other:testItemTwo," + col1.getHandle()}; + testItem.getID().toString() + ",,," + col1.getHandle(), + testItem2.getID().toString() + ",dc.identifier.other:testItemOne,," + col1.getHandle(), + testItem3.getID().toString() + ",,dc.identifier.other:testItemTwo," + col1.getHandle()}; performImportScript(csv, false); - assertRelationship(testItemTwo, testItemOne, 1, "left", 0); - assertRelationship(testItemTwo, testItemThree, 1, "left", 0); + assertRelationship(testItem2, testItem, 1, "left", 0); + assertRelationship(testItem2, testItem3, 1, "left", 0); } /** @@ -455,14 +557,17 @@ public class CSVMetadataImportReferenceIT extends AbstractEntityIntegrationTest @Test(expected = MetadataImportException.class) public void testInvalidTypeNameDefined() throws Exception { context.turnOffAuthorisationSystem(); - Item testItem = ItemBuilder.createItem(context, col1) - .withRelationshipType("Publication") - .build(); + + WorkspaceItem workspaceItem = workspaceItemService.create(context, col1, false); + Item testItem = installItemService.installItem(context, workspaceItem); + itemService.addMetadata(context, testItem, "relationship", "type", null, null, "Publication"); + itemService.update(context, testItem); + context.restoreAuthSystemState(); String[] csv = {"id,collection,relationship.type,dc.title," + "relation.isProjectOfPublication,relation.isPublicationOfProject", "+," + col1.getHandle() + ",Project,Title," + - testItem.getID().toString() + "," + testItem.getID().toString() }; + testItem.getID().toString() + "," + testItem.getID().toString()}; performImportScript(csv, true); } @@ -477,15 +582,34 @@ public class CSVMetadataImportReferenceIT extends AbstractEntityIntegrationTest } out.flush(); out.close(); + String fileLocation = csvFile.getAbsolutePath(); try { + String[] args = null; if (validateOnly) { - return runDSpaceScript("metadata-import", "-f", csvFile.getAbsolutePath(), "-s", "-v"); + args = new String[] {"metadata-import", "-f", fileLocation, "-e", eperson.getEmail(), "-s", "-v"}; } else { - return runDSpaceScript("metadata-import", "-f", csvFile.getAbsolutePath(), "-s"); + args = new String[] {"metadata-import", "-f", fileLocation, "-e", eperson.getEmail(), "-s",}; + } + TestDSpaceRunnableHandler testDSpaceRunnableHandler = new TestDSpaceRunnableHandler(); + + ScriptService scriptService = ScriptServiceFactory.getInstance().getScriptService(); + ScriptConfiguration scriptConfiguration = scriptService.getScriptConfiguration(args[0]); + + DSpaceRunnable script = null; + if (scriptConfiguration != null) { + script = scriptService.createDSpaceRunnableForScriptConfiguration(scriptConfiguration); + } + if (script != null) { + script.initialize(args, testDSpaceRunnableHandler, null); + script.run(); + } + if (testDSpaceRunnableHandler.getException() != null) { + throw testDSpaceRunnableHandler.getException(); } } finally { csvFile.delete(); } + return 0; } /** diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java index e5ef6f43a2..0934f0fd8b 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java @@ -34,7 +34,6 @@ import org.dspace.scripts.handler.DSpaceRunnableHandler; import org.dspace.scripts.service.ProcessService; import org.dspace.utils.DSpace; import org.springframework.core.task.TaskExecutor; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; /** * The {@link DSpaceRunnableHandler} dealing with Scripts started from the REST api diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CsvImportIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CsvImportIT.java index 98a673e28f..6d2855d385 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CsvImportIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CsvImportIT.java @@ -11,7 +11,6 @@ import static com.jayway.jsonpath.JsonPath.read; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.fileUpload; @@ -19,12 +18,8 @@ import static org.springframework.test.web.servlet.request.MockMvcRequestBuilder import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import java.io.BufferedWriter; import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileOutputStream; import java.io.InputStream; -import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; import java.sql.SQLException; import java.util.ArrayList; @@ -36,13 +31,11 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import com.google.gson.Gson; -import org.apache.commons.lang3.StringUtils; import org.dspace.app.rest.builder.CollectionBuilder; import org.dspace.app.rest.builder.CommunityBuilder; import org.dspace.app.rest.builder.ItemBuilder; import org.dspace.app.rest.builder.ProcessBuilder; import org.dspace.app.rest.converter.DSpaceRunnableParameterConverter; -import org.dspace.app.rest.matcher.ProcessMatcher; import org.dspace.app.rest.matcher.RelationshipMatcher; import org.dspace.app.rest.model.ParameterValueRest; import org.dspace.app.rest.projection.Projection; @@ -248,8 +241,8 @@ public class CsvImportIT extends AbstractEntityIntegrationTest { } private void performImportScript(String[] csv) throws Exception { - InputStream inputStream = new ByteArrayInputStream(String.join(System.lineSeparator(), Arrays.asList(csv)).getBytes( - StandardCharsets.UTF_8)); + InputStream inputStream = new ByteArrayInputStream(String.join(System.lineSeparator(), + Arrays.asList(csv)).getBytes(StandardCharsets.UTF_8)); MockMultipartFile bitstreamFile = new MockMultipartFile("file", "test.csv", MediaType.TEXT_PLAIN_VALUE, @@ -306,8 +299,8 @@ public class CsvImportIT extends AbstractEntityIntegrationTest { .getID().toString(); String[] csv = {"id,collection,dc.title,relationship.type,relation.isPublicationOfAuthor", csvLineString}; - InputStream inputStream = new ByteArrayInputStream(String.join(System.lineSeparator(), Arrays.asList(csv)).getBytes( - StandardCharsets.UTF_8)); + InputStream inputStream = new ByteArrayInputStream(String.join(System.lineSeparator(), + Arrays.asList(csv)).getBytes(StandardCharsets.UTF_8)); MockMultipartFile bitstreamFile = new MockMultipartFile("file", "test.csv", MediaType.TEXT_PLAIN_VALUE, From 7ea393570969737e7c7ee5ca97177086055d0c1a Mon Sep 17 00:00:00 2001 From: Raf Ponsaerts Date: Fri, 31 Jul 2020 11:42:45 +0200 Subject: [PATCH 39/96] [Task 72215] minor cleanup --- .../app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java | 1 - 1 file changed, 1 deletion(-) diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java index 0934f0fd8b..d61b87c7d4 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java @@ -231,7 +231,6 @@ public class RestDSpaceRunnableHandler implements DSpaceRunnableHandler { * @param script The script to be ran */ public void schedule(DSpaceRunnable script) { -// script.run(); TaskExecutor taskExecutor = new DSpace().getServiceManager() .getServiceByName("dspaceRunnableThreadExecutor", TaskExecutor.class); Context context = new Context(); From 1406a4c2e669b0dea1f9499eacfac033dbc7dddf Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Fri, 31 Jul 2020 17:08:23 +0200 Subject: [PATCH 40/96] Update pom.xml, ArXivService.java, and PubmedService.java --- dspace-api/pom.xml | 6 - .../dspace/submit/lookup/ArXivService.java | 162 ----------- .../dspace/submit/lookup/PubmedService.java | 274 ------------------ 3 files changed, 442 deletions(-) delete mode 100644 dspace-api/src/main/java/org/dspace/submit/lookup/ArXivService.java delete mode 100644 dspace-api/src/main/java/org/dspace/submit/lookup/PubmedService.java diff --git a/dspace-api/pom.xml b/dspace-api/pom.xml index e07e181858..7a8662bcff 100644 --- a/dspace-api/pom.xml +++ b/dspace-api/pom.xml @@ -681,12 +681,6 @@ jersey-client ${jersey.version} - - - org.glassfish.jersey.inject - jersey-hk2 - ${jersey.version} - com.amazonaws diff --git a/dspace-api/src/main/java/org/dspace/submit/lookup/ArXivService.java b/dspace-api/src/main/java/org/dspace/submit/lookup/ArXivService.java deleted file mode 100644 index 337fb4175a..0000000000 --- a/dspace-api/src/main/java/org/dspace/submit/lookup/ArXivService.java +++ /dev/null @@ -1,162 +0,0 @@ -/** - * The contents of this file are subject to the license and copyright - * detailed in the LICENSE and NOTICE files at the root of the source - * tree and available online at - * - * http://www.dspace.org/license/ - */ -package org.dspace.submit.lookup; - -import java.io.IOException; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; - -import gr.ekt.bte.core.Record; -import org.apache.commons.lang3.StringUtils; -import org.apache.http.HttpException; -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.StatusLine; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.params.CoreConnectionPNames; -import org.apache.http.params.HttpParams; -import org.dspace.app.util.XMLUtils; -import org.w3c.dom.Document; -import org.w3c.dom.Element; - -/** - * @author Andrea Bollini - * @author Kostas Stamatis - * @author Luigi Andrea Pascarelli - * @author Panagiotis Koutsourakis - */ -public class ArXivService { - private int timeout = 1000; - - /** - * How long to wait for a connection to be established. - * - * @param timeout milliseconds - */ - public void setTimeout(int timeout) { - this.timeout = timeout; - } - - public List getByDOIs(Set dois) throws HttpException, - IOException { - if (dois != null && dois.size() > 0) { - String doisQuery = StringUtils.join(dois.iterator(), " OR "); - return search(doisQuery, null, 100); - } - return null; - } - - public List searchByTerm(String title, String author, int year) - throws HttpException, IOException { - StringBuffer query = new StringBuffer(); - if (StringUtils.isNotBlank(title)) { - query.append("ti:\"").append(title).append("\""); - } - if (StringUtils.isNotBlank(author)) { - // [FAU] - if (query.length() > 0) { - query.append(" AND "); - } - query.append("au:\"").append(author).append("\""); - } - return search(query.toString(), "", 10); - } - - protected List search(String query, String arxivid, int max_result) - throws IOException, HttpException { - List results = new ArrayList(); - HttpGet method = null; - try { - HttpClient client = new DefaultHttpClient(); - HttpParams params = client.getParams(); - params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, timeout); - - try { - URIBuilder uriBuilder = new URIBuilder("http://export.arxiv.org/api/query"); - uriBuilder.addParameter("id_list", arxivid); - uriBuilder.addParameter("search_query", query); - uriBuilder.addParameter("max_results", String.valueOf(max_result)); - method = new HttpGet(uriBuilder.build()); - } catch (URISyntaxException ex) { - throw new HttpException(ex.getMessage()); - } - - // Execute the method. - HttpResponse response = client.execute(method); - StatusLine responseStatus = response.getStatusLine(); - int statusCode = responseStatus.getStatusCode(); - - if (statusCode != HttpStatus.SC_OK) { - if (statusCode == HttpStatus.SC_BAD_REQUEST) { - throw new RuntimeException("arXiv query is not valid"); - } else { - throw new RuntimeException("Http call failed: " - + responseStatus); - } - } - - try { - DocumentBuilderFactory factory = DocumentBuilderFactory - .newInstance(); - factory.setValidating(false); - factory.setIgnoringComments(true); - factory.setIgnoringElementContentWhitespace(true); - // disallow DTD parsing to ensure no XXE attacks can occur. - // See https://cheatsheetseries.owasp.org/cheatsheets/XML_External_Entity_Prevention_Cheat_Sheet.html - factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); - - DocumentBuilder db = factory.newDocumentBuilder(); - Document inDoc = db.parse(response.getEntity().getContent()); - - Element xmlRoot = inDoc.getDocumentElement(); - List dataRoots = XMLUtils.getElementList(xmlRoot, - "entry"); - - for (Element dataRoot : dataRoots) { - Record crossitem = ArxivUtils - .convertArxixDomToRecord(dataRoot); - if (crossitem != null) { - results.add(crossitem); - } - } - } catch (Exception e) { - throw new RuntimeException( - "ArXiv identifier is not valid or not exist"); - } - } finally { - if (method != null) { - method.releaseConnection(); - } - } - - return results; - } - - public Record getByArXivIDs(String raw) throws HttpException, IOException { - if (StringUtils.isNotBlank(raw)) { - raw = raw.trim(); - if (raw.startsWith("http://arxiv.org/abs/")) { - raw = raw.substring("http://arxiv.org/abs/".length()); - } else if (raw.toLowerCase().startsWith("arxiv:")) { - raw = raw.substring("arxiv:".length()); - } - List result = search("", raw, 1); - if (result != null && result.size() > 0) { - return result.get(0); - } - } - return null; - } -} diff --git a/dspace-api/src/main/java/org/dspace/submit/lookup/PubmedService.java b/dspace-api/src/main/java/org/dspace/submit/lookup/PubmedService.java deleted file mode 100644 index a5e74322f5..0000000000 --- a/dspace-api/src/main/java/org/dspace/submit/lookup/PubmedService.java +++ /dev/null @@ -1,274 +0,0 @@ -/** - * The contents of this file are subject to the license and copyright - * detailed in the LICENSE and NOTICE files at the root of the source - * tree and available online at - * - * http://www.dspace.org/license/ - */ -package org.dspace.submit.lookup; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.List; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; - -import gr.ekt.bte.core.Record; -import org.apache.commons.lang3.StringUtils; -import org.apache.http.HttpException; -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.StatusLine; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.params.CoreConnectionPNames; -import org.apache.logging.log4j.Logger; -import org.dspace.app.util.XMLUtils; -import org.dspace.core.ConfigurationManager; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.xml.sax.SAXException; - -/** - * @author Andrea Bollini - * @author Kostas Stamatis - * @author Luigi Andrea Pascarelli - * @author Panagiotis Koutsourakis - */ -public class PubmedService { - - private static final Logger log = org.apache.logging.log4j.LogManager.getLogger(PubmedService.class); - - protected int timeout = 1000; - - public void setTimeout(int timeout) { - this.timeout = timeout; - } - - public Record getByPubmedID(String pubmedid) throws HttpException, - IOException, ParserConfigurationException, SAXException { - List ids = new ArrayList(); - ids.add(pubmedid.trim()); - List items = getByPubmedIDs(ids); - if (items != null && items.size() > 0) { - return items.get(0); - } - return null; - } - - public List search(String title, String author, int year) - throws HttpException, IOException { - StringBuffer query = new StringBuffer(); - if (StringUtils.isNotBlank(title)) { - query.append("((").append(title).append("[TI]) OR ("); - // [TI] does not always work, book chapter title - query.append("(").append(title).append("[book]))"); - } - if (StringUtils.isNotBlank(author)) { - // [FAU] - if (query.length() > 0) { - query.append(" AND "); - } - query.append("(").append(author).append("[AU])"); - } - if (year != -1) { - // [DP] - if (query.length() > 0) { - query.append(" AND "); - } - query.append(year).append("[DP]"); - } - return search(query.toString()); - } - - public List search(String query) throws IOException, HttpException { - List results = new ArrayList<>(); - if (!ConfigurationManager.getBooleanProperty(SubmissionLookupService.CFG_MODULE, "remoteservice.demo")) { - HttpGet method = null; - try { - HttpClient client = new DefaultHttpClient(); - client.getParams().setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, timeout); - - URIBuilder uriBuilder = new URIBuilder( - "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi"); - uriBuilder.addParameter("db", "pubmed"); - uriBuilder.addParameter("datetype", "edat"); - uriBuilder.addParameter("retmax", "10"); - uriBuilder.addParameter("term", query); - method = new HttpGet(uriBuilder.build()); - - // Execute the method. - HttpResponse response = client.execute(method); - StatusLine statusLine = response.getStatusLine(); - int statusCode = statusLine.getStatusCode(); - - if (statusCode != HttpStatus.SC_OK) { - throw new RuntimeException("WS call failed: " - + statusLine); - } - - DocumentBuilderFactory factory = DocumentBuilderFactory - .newInstance(); - factory.setValidating(false); - factory.setIgnoringComments(true); - factory.setIgnoringElementContentWhitespace(true); - // disallow DTD parsing to ensure no XXE attacks can occur. - // See https://cheatsheetseries.owasp.org/cheatsheets/XML_External_Entity_Prevention_Cheat_Sheet.html - factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); - - DocumentBuilder builder; - try { - builder = factory.newDocumentBuilder(); - - Document inDoc = builder.parse(response.getEntity().getContent()); - - Element xmlRoot = inDoc.getDocumentElement(); - Element idList = XMLUtils.getSingleElement(xmlRoot, - "IdList"); - List pubmedIDs = XMLUtils.getElementValueList( - idList, "Id"); - results = getByPubmedIDs(pubmedIDs); - } catch (ParserConfigurationException e1) { - log.error(e1.getMessage(), e1); - } catch (SAXException e1) { - log.error(e1.getMessage(), e1); - } - } catch (Exception e1) { - log.error(e1.getMessage(), e1); - } finally { - if (method != null) { - method.releaseConnection(); - } - } - } else { - InputStream stream = null; - try { - File file = new File( - ConfigurationManager.getProperty("dspace.dir") - + "/config/crosswalks/demo/pubmed-search.xml"); - stream = new FileInputStream(file); - DocumentBuilderFactory factory = DocumentBuilderFactory - .newInstance(); - factory.setValidating(false); - factory.setIgnoringComments(true); - factory.setIgnoringElementContentWhitespace(true); - // disallow DTD parsing to ensure no XXE attacks can occur. - // See https://cheatsheetseries.owasp.org/cheatsheets/XML_External_Entity_Prevention_Cheat_Sheet.html - factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); - - DocumentBuilder builder = factory.newDocumentBuilder(); - Document inDoc = builder.parse(stream); - - Element xmlRoot = inDoc.getDocumentElement(); - Element idList = XMLUtils.getSingleElement(xmlRoot, "IdList"); - List pubmedIDs = XMLUtils.getElementValueList(idList, - "Id"); - results = getByPubmedIDs(pubmedIDs); - } catch (Exception e) { - throw new RuntimeException(e.getMessage(), e); - } finally { - if (stream != null) { - try { - stream.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - } - return results; - } - - public List getByPubmedIDs(List pubmedIDs) - throws HttpException, IOException, ParserConfigurationException, - SAXException { - List results = new ArrayList(); - HttpGet method = null; - try { - HttpClient client = new DefaultHttpClient(); - client.getParams().setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 5 * timeout); - - try { - URIBuilder uriBuilder = new URIBuilder( - "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi"); - uriBuilder.addParameter("db", "pubmed"); - uriBuilder.addParameter("retmode", "xml"); - uriBuilder.addParameter("rettype", "full"); - uriBuilder.addParameter("id", StringUtils.join( - pubmedIDs.iterator(), ",")); - method = new HttpGet(uriBuilder.build()); - } catch (URISyntaxException ex) { - throw new RuntimeException("Request not sent", ex); - } - - // Execute the method. - HttpResponse response = client.execute(method); - StatusLine statusLine = response.getStatusLine(); - int statusCode = statusLine.getStatusCode(); - - if (statusCode != HttpStatus.SC_OK) { - throw new RuntimeException("WS call failed: " + statusLine); - } - - DocumentBuilderFactory factory = DocumentBuilderFactory - .newInstance(); - factory.setValidating(false); - factory.setIgnoringComments(true); - factory.setIgnoringElementContentWhitespace(true); - // disallow DTD parsing to ensure no XXE attacks can occur. - // See https://cheatsheetseries.owasp.org/cheatsheets/XML_External_Entity_Prevention_Cheat_Sheet.html - factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); - - DocumentBuilder builder = factory.newDocumentBuilder(); - Document inDoc = builder - .parse(response.getEntity().getContent()); - - Element xmlRoot = inDoc.getDocumentElement(); - List pubArticles = XMLUtils.getElementList(xmlRoot, - "PubmedArticle"); - - for (Element xmlArticle : pubArticles) { - Record pubmedItem = null; - try { - pubmedItem = PubmedUtils - .convertPubmedDomToRecord(xmlArticle); - results.add(pubmedItem); - } catch (Exception e) { - throw new RuntimeException( - "PubmedID is not valid or not exist: " - + e.getMessage(), e); - } - } - - return results; - } finally { - if (method != null) { - method.releaseConnection(); - } - } - } - - public List search(String doi, String pmid) throws HttpException, - IOException { - StringBuffer query = new StringBuffer(); - if (StringUtils.isNotBlank(doi)) { - query.append(doi); - query.append("[AID]"); - } - if (StringUtils.isNotBlank(pmid)) { - // [FAU] - if (query.length() > 0) { - query.append(" OR "); - } - query.append(pmid).append("[PMID]"); - } - return search(query.toString()); - } -} From e35084cf239067f1118fcca5e3793aeda264d220 Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Fri, 31 Jul 2020 17:13:18 +0200 Subject: [PATCH 41/96] rollback dspace.cfg changes --- dspace/config/dspace.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dspace/config/dspace.cfg b/dspace/config/dspace.cfg index efe61c3a58..1bbd895131 100644 --- a/dspace/config/dspace.cfg +++ b/dspace/config/dspace.cfg @@ -20,7 +20,7 @@ # DSpace installation directory # Windows note: Please remember to use forward slashes for all paths (e.g. C:/dspace) -dspace.dir = /home/pasquale/Contenuti/4Science/InstallDSpace7Orig/ +dspace.dir = /dspace # URL of DSpace backend ('server' webapp). Include port number etc. # This is where REST API and all enabled server modules (OAI-PMH, SWORD, SWORDv2, RDF, etc) will respond @@ -50,7 +50,7 @@ solr.server = http://localhost:8983/solr # URL for connecting to database # * Postgres template: jdbc:postgresql://localhost:5432/dspace # * Oracle template: jdbc:oracle:thin:@//localhost:1521/xe -db.url = jdbc:postgresql://localhost:5432/dspace7 +db.url = jdbc:postgresql://localhost:5432/dspace # JDBC Driver # * For Postgres: org.postgresql.Driver @@ -63,7 +63,7 @@ db.driver = org.postgresql.Driver db.dialect = org.dspace.storage.rdbms.hibernate.postgres.DSpacePostgreSQL82Dialect # Database username and password -db.username = dspace7 +db.username = dspace db.password = dspace # Database Schema name From 774e3893f9806b0da4f6c9606cd0e21f6d016932 Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Mon, 3 Aug 2020 14:22:48 +0200 Subject: [PATCH 42/96] Add fix from DS-4530 --- dspace-api/pom.xml | 8 ++++++++ .../org/dspace/app/rest/converter/ConverterService.java | 6 +++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/dspace-api/pom.xml b/dspace-api/pom.xml index 7a8662bcff..c6da39b268 100644 --- a/dspace-api/pom.xml +++ b/dspace-api/pom.xml @@ -323,6 +323,14 @@ apache-jena-libs pom + + + + org.glassfish.jersey.inject + jersey-hk2 + ${jersey.version} + + commons-codec commons-codec diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/ConverterService.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/ConverterService.java index 563f2045ca..84ce1a0032 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/ConverterService.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/ConverterService.java @@ -34,6 +34,7 @@ import org.dspace.app.rest.security.DSpacePermissionEvaluator; import org.dspace.app.rest.security.WebSecurityExpressionEvaluator; import org.dspace.app.rest.utils.Utils; import org.dspace.services.RequestService; +import org.springframework.aop.support.AopUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.config.BeanDefinition; import org.springframework.context.annotation.ClassPathScanningCandidateComponentProvider; @@ -51,6 +52,8 @@ import org.springframework.stereotype.Service; /** * Converts domain objects from the DSpace service layer to rest objects, and from rest objects to resource * objects, applying {@link Projection}s where applicable. + * + * @author Luca Giamminonni (luca.giamminonni at 4science dot it) */ @Service public class ConverterService { @@ -150,7 +153,8 @@ public class ConverterService { .getResourceRepositoryByCategoryAndModel(baseObjectRest.getCategory(), baseObjectRest.getType()); Annotation preAuthorize = null; int maxDepth = 0; - for (Method m : repositoryToUse.getClass().getMethods()) { + // DS-4530 exclude the AOP Proxy from determining the annotations + for (Method m : AopUtils.getTargetClass(repositoryToUse).getMethods()) { if (StringUtils.equalsIgnoreCase(m.getName(), "findOne")) { int depth = howManySuperclass(m.getDeclaringClass()); if (depth > maxDepth) { From 9ee5762c3492ee4bd9edfb3f650b88d70619fb6c Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Mon, 3 Aug 2020 15:20:50 +0200 Subject: [PATCH 43/96] return 400 in external source if no record were found --- .../dspace/external/provider/impl/LiveImportDataProvider.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dspace-api/src/main/java/org/dspace/external/provider/impl/LiveImportDataProvider.java b/dspace-api/src/main/java/org/dspace/external/provider/impl/LiveImportDataProvider.java index 7bc9766a19..613c8a4fa7 100644 --- a/dspace-api/src/main/java/org/dspace/external/provider/impl/LiveImportDataProvider.java +++ b/dspace-api/src/main/java/org/dspace/external/provider/impl/LiveImportDataProvider.java @@ -127,6 +127,10 @@ public class LiveImportDataProvider implements ExternalDataProvider { * @return */ private ExternalDataObject getExternalDataObject(ImportRecord record) { + //return 400 if no record were found + if (record == null) { + throw new IllegalArgumentException("No record found for query or id"); + } ExternalDataObject externalDataObject = new ExternalDataObject(sourceIdentifier); String id = getFirstValue(record, recordIdMetadata); String display = getFirstValue(record, displayMetadata); From bdcf064660b471b37061b33eb90b5927362c5e14 Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Mon, 3 Aug 2020 15:21:30 +0200 Subject: [PATCH 44/96] return 400 on arxiv in no record were found --- .../ArXivImportMetadataSourceServiceImpl.java | 66 ++++++++++++------- 1 file changed, 41 insertions(+), 25 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java index 869aaecd36..94b31fde96 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java @@ -199,16 +199,20 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata } Invocation.Builder invocationBuilder = local.request(MediaType.TEXT_PLAIN_TYPE); Response response = invocationBuilder.get(); - String responseString = response.readEntity(String.class); - OMXMLParserWrapper records = OMXMLBuilderFactory.createOMBuilder(new StringReader(responseString)); - OMElement element = records.getDocumentElement(); - AXIOMXPath xpath = null; - try { - xpath = new AXIOMXPath("opensearch:totalResults"); - xpath.addNamespace("opensearch", "http://a9.com/-/spec/opensearch/1.1/"); - OMElement count = (OMElement) xpath.selectSingleNode(element); - return Integer.parseInt(count.getText()); - } catch (JaxenException e) { + if (response.getStatus() == 200) { + String responseString = response.readEntity(String.class); + OMXMLParserWrapper records = OMXMLBuilderFactory.createOMBuilder(new StringReader(responseString)); + OMElement element = records.getDocumentElement(); + AXIOMXPath xpath = null; + try { + xpath = new AXIOMXPath("opensearch:totalResults"); + xpath.addNamespace("opensearch", "http://a9.com/-/spec/opensearch/1.1/"); + OMElement count = (OMElement) xpath.selectSingleNode(element); + return Integer.parseInt(count.getText()); + } catch (JaxenException e) { + return null; + } + } else { return null; } } @@ -253,12 +257,16 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata } Invocation.Builder invocationBuilder = local.request(MediaType.TEXT_PLAIN_TYPE); Response response = invocationBuilder.get(); - String responseString = response.readEntity(String.class); - List omElements = splitToRecords(responseString); - for (OMElement record : omElements) { - results.add(transformSourceRecords(record)); + if (response.getStatus() == 200) { + String responseString = response.readEntity(String.class); + List omElements = splitToRecords(responseString); + for (OMElement record : omElements) { + results.add(transformSourceRecords(record)); + } + return results; + } else { + return null; } - return results; } } @@ -295,12 +303,16 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata WebTarget local = webTarget.queryParam("id_list", arxivid); Invocation.Builder invocationBuilder = local.request(MediaType.TEXT_PLAIN_TYPE); Response response = invocationBuilder.get(); - String responseString = response.readEntity(String.class); - List omElements = splitToRecords(responseString); - for (OMElement record : omElements) { - results.add(transformSourceRecords(record)); + if (response.getStatus() == 200) { + String responseString = response.readEntity(String.class); + List omElements = splitToRecords(responseString); + for (OMElement record : omElements) { + results.add(transformSourceRecords(record)); + } + return results; + } else { + return null; } - return results; } } @@ -327,12 +339,16 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata WebTarget local = webTarget.queryParam("search_query", queryString); Invocation.Builder invocationBuilder = local.request(MediaType.TEXT_PLAIN_TYPE); Response response = invocationBuilder.get(); - String responseString = response.readEntity(String.class); - List omElements = splitToRecords(responseString); - for (OMElement record : omElements) { - results.add(transformSourceRecords(record)); + if (response.getStatus() == 200) { + String responseString = response.readEntity(String.class); + List omElements = splitToRecords(responseString); + for (OMElement record : omElements) { + results.add(transformSourceRecords(record)); + } + return results; + } else { + return null; } - return results; } private String getQuery(Query query) { From 4e48ff8de12d53cbc17c7513a75d651b851223d5 Mon Sep 17 00:00:00 2001 From: Marie Verdonck Date: Thu, 6 Aug 2020 10:12:28 +0200 Subject: [PATCH 45/96] 72387: Workflow minor fixes --- .../org/dspace/content/CollectionServiceImpl.java | 3 +-- .../org/dspace/content/WorkspaceItemServiceImpl.java | 7 ++++++- .../dspace/xmlworkflow/XmlWorkflowFactoryImpl.java | 4 ++-- .../xmlworkflow/factory/XmlWorkflowFactory.java | 2 +- .../main/java/org/dspace/xmlworkflow/state/Step.java | 2 +- .../dspace/xmlworkflow/XmlWorkflowFactoryTest.java | 4 ++-- .../WorkflowDefinitionCollectionsLinkRepository.java | 12 ++---------- .../app/rest/WorkflowDefinitionRestRepositoryIT.java | 2 +- 8 files changed, 16 insertions(+), 20 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/content/CollectionServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/CollectionServiceImpl.java index 34bf4f5fc1..559b95edb8 100644 --- a/dspace-api/src/main/java/org/dspace/content/CollectionServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/CollectionServiceImpl.java @@ -57,7 +57,6 @@ import org.dspace.harvest.HarvestedCollection; import org.dspace.harvest.service.HarvestedCollectionService; import org.dspace.workflow.factory.WorkflowServiceFactory; import org.dspace.xmlworkflow.WorkflowConfigurationException; -import org.dspace.xmlworkflow.XmlWorkflowFactoryImpl; import org.dspace.xmlworkflow.factory.XmlWorkflowFactory; import org.dspace.xmlworkflow.state.Workflow; import org.dspace.xmlworkflow.storedcomponents.CollectionRole; @@ -387,7 +386,7 @@ public class CollectionServiceImpl extends DSpaceObjectServiceImpl i log.error(LogManager.getHeader(context, "setWorkflowGroup", "collection_id=" + collection.getID() + " " + e.getMessage()), e); } - if (!StringUtils.equals(XmlWorkflowFactoryImpl.LEGACY_WORKFLOW_NAME, workflow.getID())) { + if (!StringUtils.equals(workflowFactory.getDefaultWorkflow().getID(), workflow.getID())) { throw new IllegalArgumentException( "setWorkflowGroup can be used only on collection with the default basic dspace workflow. " + "Instead, the collection: " diff --git a/dspace-api/src/main/java/org/dspace/content/WorkspaceItemServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/WorkspaceItemServiceImpl.java index c45f6c737c..0d5ba53794 100644 --- a/dspace-api/src/main/java/org/dspace/content/WorkspaceItemServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/WorkspaceItemServiceImpl.java @@ -265,7 +265,12 @@ public class WorkspaceItemServiceImpl implements WorkspaceItemService { // Need to delete the workspaceitem row first since it refers // to item ID - workspaceItem.getSupervisorGroups().clear(); + try { + workspaceItem.getSupervisorGroups().clear(); + } catch (Exception e) { + log.error("failed to clear supervisor group", e); + } + workspaceItemDAO.delete(context, workspaceItem); } diff --git a/dspace-api/src/main/java/org/dspace/xmlworkflow/XmlWorkflowFactoryImpl.java b/dspace-api/src/main/java/org/dspace/xmlworkflow/XmlWorkflowFactoryImpl.java index ffc62dcddb..4150d84d04 100644 --- a/dspace-api/src/main/java/org/dspace/xmlworkflow/XmlWorkflowFactoryImpl.java +++ b/dspace-api/src/main/java/org/dspace/xmlworkflow/XmlWorkflowFactoryImpl.java @@ -97,7 +97,7 @@ public class XmlWorkflowFactoryImpl implements XmlWorkflowFactory { } @Override - public List getCollectionHandlesMappedToWorklow(Context context, String workflowName) { + public List getCollectionHandlesMappedToWorkflow(Context context, String workflowName) { List collectionsMapped = new ArrayList<>(); for (String handle : this.workflowMapping.keySet()) { if (this.workflowMapping.get(handle).getID().equals(workflowName)) { @@ -107,7 +107,7 @@ public class XmlWorkflowFactoryImpl implements XmlWorkflowFactory { collectionsMapped.add(collection); } } catch (SQLException e) { - log.error("SQLException in XmlWorkflowFactoryImpl.getCollectionHandlesMappedToWorklow trying to " + + log.error("SQLException in XmlWorkflowFactoryImpl.getCollectionHandlesMappedToWorkflow trying to " + "retrieve collection with handle: " + handle, e); } } diff --git a/dspace-api/src/main/java/org/dspace/xmlworkflow/factory/XmlWorkflowFactory.java b/dspace-api/src/main/java/org/dspace/xmlworkflow/factory/XmlWorkflowFactory.java index 5d33843747..db856bb57b 100644 --- a/dspace-api/src/main/java/org/dspace/xmlworkflow/factory/XmlWorkflowFactory.java +++ b/dspace-api/src/main/java/org/dspace/xmlworkflow/factory/XmlWorkflowFactory.java @@ -86,7 +86,7 @@ public interface XmlWorkflowFactory { * @param workflowName Name of workflow we want the collections of that are mapped to is * @return List of collections mapped to the requested workflow */ - public List getCollectionHandlesMappedToWorklow(Context context, String workflowName); + public List getCollectionHandlesMappedToWorkflow(Context context, String workflowName); /** * Returns list of collections that are not mapped to any configured workflow, and thus use the default workflow diff --git a/dspace-api/src/main/java/org/dspace/xmlworkflow/state/Step.java b/dspace-api/src/main/java/org/dspace/xmlworkflow/state/Step.java index a982107d78..16befc2626 100644 --- a/dspace-api/src/main/java/org/dspace/xmlworkflow/state/Step.java +++ b/dspace-api/src/main/java/org/dspace/xmlworkflow/state/Step.java @@ -81,7 +81,7 @@ public class Step implements BeanNameAware { /** * Get the next step based on out the outcome * @param outcome the outcome of the previous step - * @return the next stepp or NULL if there is no step configured for this outcome + * @return the next step or NULL if there is no step configured for this outcome */ public Step getNextStep(int outcome) { return outcomes.get(outcome); diff --git a/dspace-api/src/test/java/org/dspace/xmlworkflow/XmlWorkflowFactoryTest.java b/dspace-api/src/test/java/org/dspace/xmlworkflow/XmlWorkflowFactoryTest.java index c7239f1f5a..03a6a0e949 100644 --- a/dspace-api/src/test/java/org/dspace/xmlworkflow/XmlWorkflowFactoryTest.java +++ b/dspace-api/src/test/java/org/dspace/xmlworkflow/XmlWorkflowFactoryTest.java @@ -116,12 +116,12 @@ public class XmlWorkflowFactoryTest extends AbstractUnitTest { @Test public void workflowMapping_NonMappedCollection() throws WorkflowConfigurationException { Workflow workflow = xmlWorkflowFactory.getWorkflow(this.nonMappedCollection); - assertEquals("defaultWorkflow", workflow.getID()); + assertEquals(XmlWorkflowFactoryImpl.LEGACY_WORKFLOW_NAME, workflow.getID()); } @Test public void workflowMapping_MappedCollection() throws WorkflowConfigurationException { Workflow workflow = xmlWorkflowFactory.getWorkflow(this.mappedCollection); - assertEquals("selectSingleReviewer", workflow.getID()); + assertEquals( "selectSingleReviewer", workflow.getID()); } } diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/WorkflowDefinitionCollectionsLinkRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/WorkflowDefinitionCollectionsLinkRepository.java index 7ae5f5ecc0..fd1192e0bb 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/WorkflowDefinitionCollectionsLinkRepository.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/WorkflowDefinitionCollectionsLinkRepository.java @@ -12,13 +12,11 @@ import java.util.List; import javax.annotation.Nullable; import javax.servlet.http.HttpServletRequest; -import org.dspace.app.rest.converter.ConverterService; import org.dspace.app.rest.model.CollectionRest; import org.dspace.app.rest.model.WorkflowDefinitionRest; import org.dspace.app.rest.projection.Projection; import org.dspace.app.rest.repository.AbstractDSpaceRestRepository; import org.dspace.app.rest.repository.LinkRestRepository; -import org.dspace.app.rest.utils.Utils; import org.dspace.content.Collection; import org.dspace.core.Context; import org.dspace.xmlworkflow.factory.XmlWorkflowFactory; @@ -43,12 +41,6 @@ public class WorkflowDefinitionCollectionsLinkRepository extends AbstractDSpaceR @Autowired protected XmlWorkflowFactory xmlWorkflowFactory; - @Autowired - protected ConverterService converter; - - @Autowired - protected Utils utils; - /** * GET endpoint that returns the list of collections that make an explicit use of the workflow-definition. * If a collection doesn't specify the workflow-definition to be used, the default mapping applies, @@ -69,10 +61,10 @@ public class WorkflowDefinitionCollectionsLinkRepository extends AbstractDSpaceR if (xmlWorkflowFactory.isDefaultWorkflow(workflowName)) { collectionsMappedToWorkflow.addAll(xmlWorkflowFactory.getAllNonMappedCollectionsHandles(context)); } - collectionsMappedToWorkflow.addAll(xmlWorkflowFactory.getCollectionHandlesMappedToWorklow(context, + collectionsMappedToWorkflow.addAll(xmlWorkflowFactory.getCollectionHandlesMappedToWorkflow(context, workflowName)); Pageable pageable = optionalPageable != null ? optionalPageable : PageRequest.of(0, 20); - return converter.toRestPage(collectionsMappedToWorkflow, pageable, + return super.converter.toRestPage(collectionsMappedToWorkflow, pageable, projection); } else { throw new ResourceNotFoundException("No workflow with name " + workflowName + " is configured"); diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkflowDefinitionRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkflowDefinitionRestRepositoryIT.java index ab99e1d4c2..3f7ae74000 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkflowDefinitionRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkflowDefinitionRestRepositoryIT.java @@ -349,7 +349,7 @@ public class WorkflowDefinitionRestRepositoryIT extends AbstractControllerIntegr if (StringUtils.isNotBlank(firstNonDefaultWorkflowName)) { List mappedCollections - = xmlWorkflowFactory.getCollectionHandlesMappedToWorklow(context, firstNonDefaultWorkflowName); + = xmlWorkflowFactory.getCollectionHandlesMappedToWorkflow(context, firstNonDefaultWorkflowName); //When we call this facets endpoint if (mappedCollections.size() > 0) { //returns array of collection jsons that are mapped to given workflow From 362064f76f516c289f4c80eb7509a5f3e7cf8b67 Mon Sep 17 00:00:00 2001 From: Raf Ponsaerts Date: Thu, 6 Aug 2020 11:10:00 +0200 Subject: [PATCH 46/96] [Task 72215] fixing issues after merge --- .../src/test/java/org/dspace/app/rest/csv/CsvImportIT.java | 1 + .../src/test/java/org/dspace/curate/CurationScriptIT.java | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CsvImportIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CsvImportIT.java index ebe519a9b5..9f1f24f83f 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CsvImportIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CsvImportIT.java @@ -39,6 +39,7 @@ import org.dspace.app.rest.test.AbstractEntityIntegrationTest; import org.dspace.builder.CollectionBuilder; import org.dspace.builder.CommunityBuilder; import org.dspace.builder.ItemBuilder; +import org.dspace.builder.ProcessBuilder; import org.dspace.content.Collection; import org.dspace.content.Community; import org.dspace.content.Item; diff --git a/dspace-server-webapp/src/test/java/org/dspace/curate/CurationScriptIT.java b/dspace-server-webapp/src/test/java/org/dspace/curate/CurationScriptIT.java index 58a92f7505..facecc6cdb 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/curate/CurationScriptIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/curate/CurationScriptIT.java @@ -361,7 +361,7 @@ public class CurationScriptIT extends AbstractControllerIntegrationTest { .andExpect(jsonPath("$", is( ProcessMatcher.matchProcess("curate", String.valueOf(admin.getID()), parameters, - ProcessStatus.SCHEDULED)))) + ProcessStatus.COMPLETED)))) .andDo(result -> idRef .set(read(result.getResponse().getContentAsString(), "$.processId"))); } finally { @@ -414,7 +414,7 @@ public class CurationScriptIT extends AbstractControllerIntegrationTest { .andExpect(jsonPath("$", is( ProcessMatcher.matchProcess("curate", String.valueOf(admin.getID()), parameters, - ProcessStatus.SCHEDULED)))) + ProcessStatus.COMPLETED)))) .andDo(result -> idRef .set(read(result.getResponse().getContentAsString(), "$.processId"))); } finally { From 3d087aa18443a2f2a76ac23fc393f737145651b1 Mon Sep 17 00:00:00 2001 From: Marie Verdonck Date: Thu, 6 Aug 2020 13:52:14 +0200 Subject: [PATCH 47/96] 72426: Sitemap cron schedule configurable in dspace.cfg --- .../java/org/dspace/app/rest/SitemapRestControllerIT.java | 2 +- dspace/config/dspace.cfg | 5 +++++ dspace/config/spring/api/task-scheduler.xml | 4 ++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/SitemapRestControllerIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/SitemapRestControllerIT.java index b428435ede..c6de3cdbaa 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/SitemapRestControllerIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/SitemapRestControllerIT.java @@ -80,7 +80,7 @@ public class SitemapRestControllerIT extends AbstractControllerIntegrationTest { public void testSitemap_notValidSiteMapFile() throws Exception { //** WHEN ** //We attempt to retrieve a non valid sitemap file - getClient().perform(get("/" + SITEMAPS_ENDPOINT + "/notValidSiteMapFile")) + getClient().perform(get("/" + SITEMAPS_ENDPOINT + "/no-such-file")) //** THEN ** .andExpect(status().isNotFound()); } diff --git a/dspace/config/dspace.cfg b/dspace/config/dspace.cfg index 6550b2e3d3..df21333945 100644 --- a/dspace/config/dspace.cfg +++ b/dspace/config/dspace.cfg @@ -1445,6 +1445,11 @@ sitemap.engineurls = http://www.google.com/webmasters/sitemaps/ping?sitemap= # # No known Sitemap 'ping' URL for MSN/Live search +# Define cron for how frequently the sitemap should refresh. +# Defaults to running daily at 1:15am +# Syntax is defined at https://www.quartz-scheduler.org/api/2.3.0/org/quartz/CronTrigger.html +sitemap.cron = 0 * * * * ? + ##### SHERPA/Romeo Integration Settings #### # the SHERPA/RoMEO endpoint sherpa.romeo.url = http://www.sherpa.ac.uk/romeo/api29.php diff --git a/dspace/config/spring/api/task-scheduler.xml b/dspace/config/spring/api/task-scheduler.xml index a77b7aafaa..d1322655a9 100644 --- a/dspace/config/spring/api/task-scheduler.xml +++ b/dspace/config/spring/api/task-scheduler.xml @@ -8,11 +8,11 @@ http://www.springframework.org/task/spring-task.xsd"> - + - + From 00166862ea65d649e21fe4e67dc09dceb50e1209 Mon Sep 17 00:00:00 2001 From: Marie Verdonck Date: Thu, 6 Aug 2020 14:03:48 +0200 Subject: [PATCH 48/96] Revert whitespace changes --- .../dspace/app/rest/ShibbolethRestControllerIT.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ShibbolethRestControllerIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ShibbolethRestControllerIT.java index fea0bafd0d..e13d091168 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ShibbolethRestControllerIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ShibbolethRestControllerIT.java @@ -26,8 +26,8 @@ public class ShibbolethRestControllerIT extends AbstractControllerIntegrationTes String token = getAuthToken(eperson.getEmail(), password); getClient(token).perform(get("/api/authn/shibboleth")) - .andExpect(status().is3xxRedirection()) - .andExpect(redirectedUrl("http://localhost:4000")); + .andExpect(status().is3xxRedirection()) + .andExpect(redirectedUrl("http://localhost:4000")); } @Test @@ -35,8 +35,8 @@ public class ShibbolethRestControllerIT extends AbstractControllerIntegrationTes String token = getAuthToken(eperson.getEmail(), password); getClient(token).perform(get("/api/authn/shibboleth") - .param("redirectUrl", "http://dspace.org")) - .andExpect(status().is3xxRedirection()) - .andExpect(redirectedUrl("http://dspace.org")); + .param("redirectUrl", "http://dspace.org")) + .andExpect(status().is3xxRedirection()) + .andExpect(redirectedUrl("http://dspace.org")); } } From cff1d0a69af562327bdc87f711ffc881f7b17b00 Mon Sep 17 00:00:00 2001 From: Marie Verdonck Date: Thu, 6 Aug 2020 14:22:26 +0200 Subject: [PATCH 49/96] poolsize fix --- dspace/config/spring/api/task-scheduler.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspace/config/spring/api/task-scheduler.xml b/dspace/config/spring/api/task-scheduler.xml index d1322655a9..da6d64730a 100644 --- a/dspace/config/spring/api/task-scheduler.xml +++ b/dspace/config/spring/api/task-scheduler.xml @@ -8,7 +8,7 @@ http://www.springframework.org/task/spring-task.xsd"> - + From 69b722a97f81f7f610cfb6cef7f6d3353eb096d4 Mon Sep 17 00:00:00 2001 From: Raf Ponsaerts Date: Thu, 6 Aug 2020 15:56:12 +0200 Subject: [PATCH 50/96] [Task 72430] removed the metadata export file parameter for the REST api --- .../dspace/app/bulkedit/MetadataExport.java | 19 ++- .../app/bulkedit/MetadataExportCli.java | 33 ++++ .../MetadataExportCliScriptConfiguration.java | 26 ++++ .../MetadataExportScriptConfiguration.java | 4 - .../config/spring/api/scripts.xml | 4 +- .../dspace/app/bulkedit/MetadataExportIT.java | 41 +++++ .../config/spring/rest/scripts.xml | 5 + .../org/dspace/app/rest/csv/CsvExportIT.java | 147 ++++++++++++++++++ dspace/config/spring/api/scripts.xml | 4 +- dspace/config/spring/rest/scripts.xml | 5 + 10 files changed, 273 insertions(+), 15 deletions(-) create mode 100644 dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExportCli.java create mode 100644 dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExportCliScriptConfiguration.java create mode 100644 dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CsvExportIT.java diff --git a/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExport.java b/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExport.java index 2e4f333820..a91b83e6d9 100644 --- a/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExport.java +++ b/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExport.java @@ -8,6 +8,7 @@ package org.dspace.app.bulkedit; import java.sql.SQLException; +import java.util.UUID; import org.apache.commons.cli.ParseException; import org.dspace.content.service.MetadataDSpaceCsvExportService; @@ -41,8 +42,7 @@ public class MetadataExport extends DSpaceRunnable { + + + @Override + public Options getOptions() { + Options options = super.getOptions(); + options.addOption("f", "file", true, "destination where you want file written"); + options.getOption("f").setType(OutputStream .class); + options.getOption("f").setRequired(true); + super.options = options; + return options; + } +} diff --git a/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExportScriptConfiguration.java b/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExportScriptConfiguration.java index 65c0ddd8cf..0c513c4667 100644 --- a/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExportScriptConfiguration.java +++ b/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExportScriptConfiguration.java @@ -7,7 +7,6 @@ */ package org.dspace.app.bulkedit; -import java.io.OutputStream; import java.sql.SQLException; import org.apache.commons.cli.Options; @@ -56,9 +55,6 @@ public class MetadataExportScriptConfiguration extends options.addOption("i", "id", true, "ID or handle of thing to export (item, collection, or community)"); options.getOption("i").setType(String.class); - options.addOption("f", "file", true, "destination where you want file written"); - options.getOption("f").setType(OutputStream.class); - options.getOption("f").setRequired(true); options.addOption("a", "all", false, "include all metadata fields that are not normally changed (e.g. provenance)"); options.getOption("a").setType(boolean.class); diff --git a/dspace-api/src/test/data/dspaceFolder/config/spring/api/scripts.xml b/dspace-api/src/test/data/dspaceFolder/config/spring/api/scripts.xml index 76cb57a40d..aab4357892 100644 --- a/dspace-api/src/test/data/dspaceFolder/config/spring/api/scripts.xml +++ b/dspace-api/src/test/data/dspaceFolder/config/spring/api/scripts.xml @@ -14,9 +14,9 @@ - + - + diff --git a/dspace-api/src/test/java/org/dspace/app/bulkedit/MetadataExportIT.java b/dspace-api/src/test/java/org/dspace/app/bulkedit/MetadataExportIT.java index a6a4780aa9..d7379351e5 100644 --- a/dspace-api/src/test/java/org/dspace/app/bulkedit/MetadataExportIT.java +++ b/dspace-api/src/test/java/org/dspace/app/bulkedit/MetadataExportIT.java @@ -13,6 +13,7 @@ import java.io.File; import java.io.FileInputStream; import java.nio.charset.StandardCharsets; +import org.apache.commons.cli.ParseException; import org.apache.commons.io.IOUtils; import org.dspace.AbstractIntegrationTestWithDatabase; import org.dspace.app.launcher.ScriptLauncher; @@ -23,12 +24,22 @@ import org.dspace.builder.ItemBuilder; import org.dspace.content.Collection; import org.dspace.content.Community; import org.dspace.content.Item; +import org.dspace.scripts.DSpaceRunnable; +import org.dspace.scripts.configuration.ScriptConfiguration; +import org.dspace.scripts.factory.ScriptServiceFactory; +import org.dspace.scripts.service.ScriptService; import org.dspace.services.ConfigurationService; import org.dspace.services.factory.DSpaceServicesFactory; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; public class MetadataExportIT extends AbstractIntegrationTestWithDatabase { + + @Rule + public ExpectedException thrown = ExpectedException.none(); + private final ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService(); @@ -59,4 +70,34 @@ public class MetadataExportIT assertTrue(fileContent.contains("Donald, Smith")); assertTrue(fileContent.contains(String.valueOf(item.getID()))); } + + @Test(expected = ParseException.class) + public void metadataExportWithoutFileParameter() + throws IllegalAccessException, InstantiationException, ParseException { + context.turnOffAuthorisationSystem(); + Community community = CommunityBuilder.createCommunity(context) + .build(); + Collection collection = CollectionBuilder.createCollection(context, community) + .build(); + Item item = ItemBuilder.createItem(context, collection) + .withAuthor("Donald, Smith") + .build(); + context.restoreAuthSystemState(); + + String[] args = new String[] {"metadata-export", + "-i", String.valueOf(item.getHandle())}; + TestDSpaceRunnableHandler testDSpaceRunnableHandler = new TestDSpaceRunnableHandler(); + + ScriptService scriptService = ScriptServiceFactory.getInstance().getScriptService(); + ScriptConfiguration scriptConfiguration = scriptService.getScriptConfiguration(args[0]); + + DSpaceRunnable script = null; + if (scriptConfiguration != null) { + script = scriptService.createDSpaceRunnableForScriptConfiguration(scriptConfiguration); + } + if (script != null) { + script.initialize(args, testDSpaceRunnableHandler, null); + script.run(); + } + } } diff --git a/dspace-server-webapp/src/test/data/dspaceFolder/config/spring/rest/scripts.xml b/dspace-server-webapp/src/test/data/dspaceFolder/config/spring/rest/scripts.xml index 7eb5ac7270..c102c99555 100644 --- a/dspace-server-webapp/src/test/data/dspaceFolder/config/spring/rest/scripts.xml +++ b/dspace-server-webapp/src/test/data/dspaceFolder/config/spring/rest/scripts.xml @@ -12,4 +12,9 @@ + + + + + \ No newline at end of file diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CsvExportIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CsvExportIT.java new file mode 100644 index 0000000000..6186b4b242 --- /dev/null +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CsvExportIT.java @@ -0,0 +1,147 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest.csv; + +import static com.jayway.jsonpath.JsonPath.read; +import static org.hamcrest.Matchers.is; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.fileUpload; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import com.google.gson.Gson; +import org.dspace.app.rest.converter.DSpaceRunnableParameterConverter; +import org.dspace.app.rest.matcher.ProcessMatcher; +import org.dspace.app.rest.model.ParameterValueRest; +import org.dspace.app.rest.projection.Projection; +import org.dspace.app.rest.test.AbstractControllerIntegrationTest; +import org.dspace.builder.CollectionBuilder; +import org.dspace.builder.CommunityBuilder; +import org.dspace.builder.ItemBuilder; +import org.dspace.builder.ProcessBuilder; +import org.dspace.content.Collection; +import org.dspace.content.Community; +import org.dspace.content.Item; +import org.dspace.content.ProcessStatus; +import org.dspace.scripts.DSpaceCommandLineParameter; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; + +public class CsvExportIT extends AbstractControllerIntegrationTest { + + + @Autowired + private DSpaceRunnableParameterConverter dSpaceRunnableParameterConverter; + + @Test + public void metadataExportTestWithoutFileParameterSucceeds() throws Exception { + + context.turnOffAuthorisationSystem(); + + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Community child1 = CommunityBuilder.createSubCommunity(context, parentCommunity) + .withName("Sub Community") + .build(); + Collection col1 = CollectionBuilder.createCollection(context, child1).withName("Collection 1").build(); + Collection col2 = CollectionBuilder.createCollection(context, child1).withName("Collection 2").build(); + Collection col3 = CollectionBuilder.createCollection(context, child1).withName("OrgUnits").build(); + + Item article = ItemBuilder.createItem(context, col1) + .withTitle("Article") + .withIssueDate("2017-10-17") + .withRelationshipType("Publication") + .build(); + + AtomicReference idRef = new AtomicReference<>(); + + LinkedList parameters = new LinkedList<>(); + parameters.add(new DSpaceCommandLineParameter("-i", col1.getHandle())); + + List list = parameters.stream() + .map(dSpaceCommandLineParameter -> dSpaceRunnableParameterConverter + .convert(dSpaceCommandLineParameter, Projection.DEFAULT)) + .collect(Collectors.toList()); + + try { + String token = getAuthToken(admin.getEmail(), password); + + getClient(token) + .perform(fileUpload("/api/system/scripts/metadata-export/processes") + .param("properties", + new Gson().toJson(list))) + .andExpect(status().isAccepted()) + .andExpect(jsonPath("$", is( + ProcessMatcher.matchProcess("metadata-export", + String.valueOf(admin.getID()), parameters, + ProcessStatus.COMPLETED)))) + .andDo(result -> idRef + .set(read(result.getResponse().getContentAsString(), "$.processId"))); + String t = ""; + } finally { + ProcessBuilder.deleteProcess(idRef.get()); + } + } + + @Test + public void metadataExportTestWithFileParameterFails() throws Exception { + + context.turnOffAuthorisationSystem(); + + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Community child1 = CommunityBuilder.createSubCommunity(context, parentCommunity) + .withName("Sub Community") + .build(); + Collection col1 = CollectionBuilder.createCollection(context, child1).withName("Collection 1").build(); + Collection col2 = CollectionBuilder.createCollection(context, child1).withName("Collection 2").build(); + Collection col3 = CollectionBuilder.createCollection(context, child1).withName("OrgUnits").build(); + + Item article = ItemBuilder.createItem(context, col1) + .withTitle("Article") + .withIssueDate("2017-10-17") + .withRelationshipType("Publication") + .build(); + + AtomicReference idRef = new AtomicReference<>(); + + LinkedList parameters = new LinkedList<>(); + parameters.add(new DSpaceCommandLineParameter("-f", "test.csv")); + parameters.add(new DSpaceCommandLineParameter("-i", col1.getHandle())); + + List list = parameters.stream() + .map(dSpaceCommandLineParameter -> dSpaceRunnableParameterConverter + .convert(dSpaceCommandLineParameter, Projection.DEFAULT)) + .collect(Collectors.toList()); + + try { + String token = getAuthToken(admin.getEmail(), password); + + getClient(token) + .perform(fileUpload("/api/system/scripts/metadata-export/processes") + .param("properties", + new Gson().toJson(list))) + .andExpect(status().isAccepted()) + .andExpect(jsonPath("$", is( + ProcessMatcher.matchProcess("metadata-export", + String.valueOf(admin.getID()), parameters, + ProcessStatus.FAILED)))) + .andDo(result -> idRef + .set(read(result.getResponse().getContentAsString(), "$.processId"))); + String t = ""; + } finally { + ProcessBuilder.deleteProcess(idRef.get()); + } + } +} diff --git a/dspace/config/spring/api/scripts.xml b/dspace/config/spring/api/scripts.xml index 31d457f09d..70814b5a82 100644 --- a/dspace/config/spring/api/scripts.xml +++ b/dspace/config/spring/api/scripts.xml @@ -14,9 +14,9 @@ - + - + diff --git a/dspace/config/spring/rest/scripts.xml b/dspace/config/spring/rest/scripts.xml index 04cacb4930..def053531f 100644 --- a/dspace/config/spring/rest/scripts.xml +++ b/dspace/config/spring/rest/scripts.xml @@ -12,4 +12,9 @@ + + + + + \ No newline at end of file From a1c9bc9061b5456b3daf526cd179216b1181943a Mon Sep 17 00:00:00 2001 From: Marie Verdonck Date: Thu, 6 Aug 2020 16:10:07 +0200 Subject: [PATCH 51/96] Collection and Community Builder moved fix --- .../org/dspace/app/rest/SitemapRestControllerIT.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/SitemapRestControllerIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/SitemapRestControllerIT.java index c6de3cdbaa..d51992e9be 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/SitemapRestControllerIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/SitemapRestControllerIT.java @@ -7,9 +7,7 @@ */ package org.dspace.app.rest; -import static org.dspace.app.rest.builder.CollectionBuilder.createCollection; -import static org.dspace.app.rest.builder.CommunityBuilder.createCommunity; -import static org.dspace.app.rest.builder.ItemBuilder.createItem; +import static org.dspace.builder.ItemBuilder.createItem; import static org.junit.Assert.assertTrue; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; @@ -18,6 +16,8 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. import javax.servlet.ServletException; import org.dspace.app.rest.test.AbstractControllerIntegrationTest; +import org.dspace.builder.CollectionBuilder; +import org.dspace.builder.CommunityBuilder; import org.dspace.content.Collection; import org.dspace.content.Community; import org.dspace.content.Item; @@ -52,8 +52,8 @@ public class SitemapRestControllerIT extends AbstractControllerIntegrationTest { context.turnOffAuthorisationSystem(); - Community community = createCommunity(context).build(); - Collection collection = createCollection(context, community).build(); + Community community = CommunityBuilder.createCommunity(context).build(); + Collection collection = CollectionBuilder.createCollection(context, community).build(); this.item1 = createItem(context, collection) .withTitle("Test 1") .withIssueDate("2010-10-17") From b7a24fc2613fc8de23604e76ec3b680580b4404e Mon Sep 17 00:00:00 2001 From: Yana De Pauw Date: Thu, 6 Aug 2020 17:16:22 +0200 Subject: [PATCH 52/96] 72428: Metadata PATCH: feedback processing --- .../content/RelationshipMetadataValue.java | 5 + .../content/service/DSpaceObjectService.java | 53 +- .../org/dspace/app/rest/PatchMetadataIT.java | 530 +++++++++++++++++- 3 files changed, 572 insertions(+), 16 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/content/RelationshipMetadataValue.java b/dspace-api/src/main/java/org/dspace/content/RelationshipMetadataValue.java index f443f89ce5..637d1c094b 100644 --- a/dspace-api/src/main/java/org/dspace/content/RelationshipMetadataValue.java +++ b/dspace-api/src/main/java/org/dspace/content/RelationshipMetadataValue.java @@ -60,6 +60,11 @@ public class RelationshipMetadataValue extends MetadataValue { return super.equals(obj); } + /** + * Retrieves the Relationship ID from which the current RelationshipMetadataValue is derived + * + * @return the relationship ID + */ public int getRelationshipId() { return Integer.parseInt(getAuthority().substring(Constants.VIRTUAL_AUTHORITY_PREFIX.length())); } diff --git a/dspace-api/src/main/java/org/dspace/content/service/DSpaceObjectService.java b/dspace-api/src/main/java/org/dspace/content/service/DSpaceObjectService.java index 941543a28e..e1eeca062e 100644 --- a/dspace-api/src/main/java/org/dspace/content/service/DSpaceObjectService.java +++ b/dspace-api/src/main/java/org/dspace/content/service/DSpaceObjectService.java @@ -200,6 +200,7 @@ public interface DSpaceObjectService { * and the ISO3166 country code. null means the * value has no language (for example, a date). * @param values the values to add. + * @return the list of MetadataValues of the object including the newly added values * @throws SQLException if database error */ public List addMetadata(Context context, T dso, String schema, String element, String qualifier, @@ -223,6 +224,7 @@ public interface DSpaceObjectService { * @param values the values to add. * @param authorities the external authority key for this value (or null) * @param confidences the authority confidence (default 0) + * @return the list of MetadataValues of the object including the newly added values * @throws SQLException if database error */ public List addMetadata(Context context, T dso, String schema, String element, String qualifier, @@ -243,8 +245,8 @@ public interface DSpaceObjectService { * @param values the values to add. * @param authorities the external authority key for this value (or null) * @param confidences the authority confidence (default 0) + * @return the list of MetadataValues of the object including the newly added values * @throws SQLException if database error - * @return */ public List addMetadata(Context context, T dso, MetadataField metadataField, String lang, List values, List authorities, List confidences) throws SQLException; @@ -252,22 +254,53 @@ public interface DSpaceObjectService { /** * Shortcut for {@link #addMetadata(Context, DSpaceObject, MetadataField, String, List, List, List)} when a single * value need to be added - * - * @param context - * @param dso - * @param metadataField - * @param language - * @param value - * @param authority - * @param confidence + * + * @param context DSpace context + * @param dso DSpaceObject + * @param metadataField the metadata field to which the value is to be set + * @param language the ISO639 language code, optionally followed by an underscore + * and the ISO3166 country code. null means the + * value has no language (for example, a date). + * @param value the value to add. + * @param authority the external authority key for this value (or null) + * @param confidence the authority confidence (default 0) + * @return the first MetadataValue of the DSpaceObject * @throws SQLException */ public MetadataValue addMetadata(Context context, T dso, MetadataField metadataField, String language, String value, String authority, int confidence) throws SQLException; + /** + * Add a metadatafield. These are appended to existing values. + * Use clearMetadata to remove values. + * + * @param context DSpace context + * @param dso DSpaceObject + * @param metadataField the metadata field to which the value is to be set + * @param language the ISO639 language code, optionally followed by an underscore + * and the ISO3166 country code. null means the + * value has no language (for example, a date). + * @param value the value to add. + * @return the first MetadataValue of the DSpaceObject + * @throws SQLException if database error + */ public MetadataValue addMetadata(Context context, T dso, MetadataField metadataField, String language, String value) throws SQLException; + /** + * Add a metadatafields. These are appended to existing values. + * Use clearMetadata to remove values. + * + * @param context DSpace context + * @param dso DSpaceObject + * @param metadataField the metadata field to which the value is to be set + * @param language the ISO639 language code, optionally followed by an underscore + * and the ISO3166 country code. null means the + * value has no language (for example, a date). + * @param values the values to add. + * @return the list of MetadataValues of the object including the newly added values + * @throws SQLException if database error + */ public List addMetadata(Context context, T dso, MetadataField metadataField, String language, List values) throws SQLException; @@ -286,6 +319,7 @@ public interface DSpaceObjectService { * and the ISO3166 country code. null means the * value has no language (for example, a date). * @param value the value to add. + * @return the first MetadataValue of the DSpaceObject * @throws SQLException if database error */ public MetadataValue addMetadata(Context context, T dso, String schema, String element, String qualifier, @@ -308,6 +342,7 @@ public interface DSpaceObjectService { * @param value the value to add. * @param authority the external authority key for this value (or null) * @param confidence the authority confidence (default 0) + * @return the first MetadataValue of the DSpaceObject * @throws SQLException if database error */ public MetadataValue addMetadata(Context context, T dso, String schema, String element, String qualifier, diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java index e1b7258ae1..1e16eefb27 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java @@ -7,9 +7,11 @@ */ package org.dspace.app.rest; +import static com.jayway.jsonpath.JsonPath.read; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.startsWith; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.patch; @@ -18,12 +20,16 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import java.io.IOException; +import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.atomic.AtomicReference; import org.dspace.app.rest.builder.CollectionBuilder; import org.dspace.app.rest.builder.CommunityBuilder; import org.dspace.app.rest.builder.ItemBuilder; +import org.dspace.app.rest.builder.RelationshipBuilder; import org.dspace.app.rest.builder.WorkspaceItemBuilder; import org.dspace.app.rest.matcher.MetadataMatcher; import org.dspace.app.rest.model.MetadataValueRest; @@ -44,11 +50,11 @@ import org.dspace.content.service.ItemService; import org.dspace.content.service.RelationshipTypeService; import org.dspace.content.service.WorkspaceItemService; import org.hamcrest.Matchers; +import org.junit.After; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.MediaType; -import org.springframework.test.web.servlet.MvcResult; /** * Created by kristof on 20/02/2020 @@ -75,6 +81,9 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { private List authorsOriginalOrder; + private AtomicReference idRef1; + private AtomicReference idRef2; + private String addedAuthor; private String replacedAuthor; @@ -94,6 +103,13 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { context.restoreAuthSystemState(); } + @After + @Override + public void destroy() throws Exception { + super.destroy(); + cleanupPersonRelations(); + } + /** * A method to create a workspace publication containing 5 authors: 3 regular authors and 2 related Person items. * The authors are added in a specific order: @@ -152,15 +168,15 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { context.restoreAuthSystemState(); // Create a relationship between publication and person 1 - MvcResult mvcResult = getClient(adminToken).perform(post("/api/core/relationships") + idRef1 = new AtomicReference<>(); + getClient(adminToken).perform(post("/api/core/relationships") .param("relationshipType", publicationPersonRelationshipType.getID().toString()) .contentType(MediaType.parseMediaType (org.springframework.data.rest.webmvc.RestMediaTypes.TEXT_URI_LIST_VALUE)) .content("https://localhost:8080/server/api/core/items/" + publicationItem.getItem().getID() + "\n" + "https://localhost:8080/server/api/core/items/" + personItem1.getID())) .andExpect(status().isCreated()) - .andReturn(); - + .andDo(result -> idRef1.set(read(result.getResponse().getContentAsString(), "$.id"))); context.turnOffAuthorisationSystem(); // Add two more regular authors @@ -175,19 +191,20 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { context.restoreAuthSystemState(); // Create a relationship between publication and person 2 - mvcResult = getClient(adminToken).perform(post("/api/core/relationships") + AtomicReference idRef2 = new AtomicReference<>(); + getClient(adminToken).perform(post("/api/core/relationships") .param("relationshipType", publicationPersonRelationshipType.getID().toString()) .contentType(MediaType.parseMediaType (org.springframework.data.rest.webmvc.RestMediaTypes.TEXT_URI_LIST_VALUE)) .content("https://localhost:8080/server/api/core/items/" + publicationItem.getItem().getID() + "\n" + "https://localhost:8080/server/api/core/items/" + personItem2.getID())) .andExpect(status().isCreated()) - .andReturn(); + .andDo(result -> idRef2.set(read(result.getResponse().getContentAsString(), "$.id"))); publication = workspaceItemService.find(context, publicationItem.getID()); List publicationAuthorList = itemService.getMetadata(publication.getItem(), "dc", "contributor", "author", Item.ANY); - assertThat(publicationAuthorList.size(), equalTo(5)); + assertEquals(publicationAuthorList.size(), 5); assertThat(publicationAuthorList.get(0).getValue(), equalTo(authorsOriginalOrder.get(0))); assertThat(publicationAuthorList.get(0).getAuthority(), not(startsWith("virtual::"))); assertThat(publicationAuthorList.get(1).getValue(), equalTo(authorsOriginalOrder.get(1))); @@ -200,6 +217,75 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { assertThat(publicationAuthorList.get(4).getAuthority(), startsWith("virtual::")); } + /** + * Clean up created Person Relationshipts + * @throws IOException + * @throws SQLException + */ + private void cleanupPersonRelations() throws IOException, SQLException { + if (idRef1 != null) { + RelationshipBuilder.deleteRelationship(idRef1.get()); + idRef1 = null; + } + if (idRef2 != null) { + RelationshipBuilder.deleteRelationship(idRef2.get()); + idRef2 = null; + } + } + + /** + * A method to create a workspace publication containing 5 authors: 3 regular authors and 2 related Person items. + * The authors are added in a specific order: + * - "Whyte, William": Regular author + * - "Dahlen, Sarah": Regular Person + * - "Peterson, Karrie": Regular author + * - "Perotti, Enrico": Regular author + * - "Linton, Oliver": Regular Person + */ + private void initPlainTextPublicationWorkspace() throws Exception { + authorsOriginalOrder = new ArrayList<>(); + authorsOriginalOrder.add("Whyte, William"); + authorsOriginalOrder.add("Dahlen, Sarah"); + authorsOriginalOrder.add("Peterson, Karrie"); + authorsOriginalOrder.add("Perotti, Enrico"); + authorsOriginalOrder.add("Linton, Oliver"); + + addedAuthor = "Semple, Robert"; + replacedAuthor = "New Value"; + + context.turnOffAuthorisationSystem(); + + publicationItem = WorkspaceItemBuilder.createWorkspaceItem(context, collection) + .withTitle("Publication 1") + .withRelationshipType("Publication") + .build(); + + String adminToken = getAuthToken(admin.getEmail(), password); + + // Make sure we grab the latest instance of the Item from the database before adding a regular author + WorkspaceItem publication = workspaceItemService.find(context, publicationItem.getID()); + itemService.addMetadata(context, publication.getItem(), + "dc", "contributor", "author", Item.ANY, authorsOriginalOrder); + workspaceItemService.update(context, publication); + + context.restoreAuthSystemState(); + + publication = workspaceItemService.find(context, publicationItem.getID()); + List publicationAuthorList = + itemService.getMetadata(publication.getItem(), "dc", "contributor", "author", Item.ANY); + assertEquals(publicationAuthorList.size(), 5); + assertThat(publicationAuthorList.get(0).getValue(), equalTo(authorsOriginalOrder.get(0))); + assertThat(publicationAuthorList.get(0).getAuthority(), not(startsWith("virtual::"))); + assertThat(publicationAuthorList.get(1).getValue(), equalTo(authorsOriginalOrder.get(1))); + assertThat(publicationAuthorList.get(1).getAuthority(), not(startsWith("virtual::"))); + assertThat(publicationAuthorList.get(2).getValue(), equalTo(authorsOriginalOrder.get(2))); + assertThat(publicationAuthorList.get(2).getAuthority(), not(startsWith("virtual::"))); + assertThat(publicationAuthorList.get(3).getValue(), equalTo(authorsOriginalOrder.get(3))); + assertThat(publicationAuthorList.get(3).getAuthority(), not(startsWith("virtual::"))); + assertThat(publicationAuthorList.get(4).getValue(), equalTo(authorsOriginalOrder.get(4))); + assertThat(publicationAuthorList.get(4).getAuthority(), not(startsWith("virtual::"))); + } + /** * This test will move an author (dc.contributor.author) within a workspace publication's "traditionalpageone" * section from position 1 to 0 using a PATCH request and verify the order of the authors within the section. @@ -611,6 +697,436 @@ public class PatchMetadataIT extends AbstractEntityIntegrationTest { removeTraditionalPageOneAuthorTest(4, expectedOrder); } + /** + * This test will move an author (dc.contributor.author) within a workspace publication's "traditionalpageone" + * section from position 1 to 0 using a PATCH request and verify the order of the authors within the section. + * This test uses only plain text authors + * Original Order: 0,1,2,3,4 + * Expected Order: 1,0,2,3,4 + */ + @Test + public void moveTraditionalPageOnePlainTextAuthorOneToZeroTest() throws Exception { + initPlainTextPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + moveTraditionalPageOneAuthorTest(1, 0, expectedOrder); + } + + /** + * This test will move an author (dc.contributor.author) within a workspace publication's "traditionalpageone" + * section from position 2 to 0 using a PATCH request and verify the order of the authors within the section. + * This test uses only plain text authors + * Original Order: 0,1,2,3,4 + * Expected Order: 2,0,1,3,4 + */ + @Test + public void moveTraditionalPageOnePlainTextAuthorTwoToZeroTest() throws Exception { + initPlainTextPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + moveTraditionalPageOneAuthorTest(2, 0, expectedOrder); + } + + /** + * This test will move an author (dc.contributor.author) within a workspace publication's "traditionalpageone" + * section from position 3 to 0 using a PATCH request and verify the order of the authors within the section. + * This test uses only plain text authors + * Original Order: 0,1,2,3,4 + * Expected Order: 3,0,1,2,4 + */ + @Test + public void moveTraditionalPageOnePlainTextAuthorThreeToZeroTest() throws Exception { + initPlainTextPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + moveTraditionalPageOneAuthorTest(3, 0, expectedOrder); + } + + /** + * This test will move an author (dc.contributor.author) within a workspace publication's "traditionalpageone" + * section from position 4 to 0 using a PATCH request and verify the order of the authors within the section. + * This test uses only plain text authors + * Original Order: 0,1,2,3,4 + * Expected Order: 4,0,1,2,3 + */ + @Test + public void moveTraditionalPageOnePlainTextAuthorFourToZeroTest() throws Exception { + initPlainTextPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(4)); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(3)); + + moveTraditionalPageOneAuthorTest(4, 0, expectedOrder); + } + + /** + * This test will move an author (dc.contributor.author) within a workspace publication's "traditionalpageone" + * section from position 1 to 3 using a PATCH request and verify the order of the authors within the section. + * This test uses only plain text authors + * Original Order: 0,1,2,3,4 + * Expected Order: 0,2,3,1,4 + */ + @Test + public void moveTraditionalPageOnePlainTextAuthorOneToThreeTest() throws Exception { + initPlainTextPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + moveTraditionalPageOneAuthorTest(1, 3, expectedOrder); + } + + /** + * This test will move an author (dc.contributor.author) within a workspace publication's "traditionalpageone" + * section from position 1 to 4 using a PATCH request and verify the order of the authors within the section. + * This test uses only plain text authors + * Original Order: 0,1,2,3,4 + * Expected Order: 0,2,3,4,1 + */ + @Test + public void moveTraditionalPageOnePlainTextAuthorOneToFourTest() throws Exception { + initPlainTextPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(4)); + expectedOrder.add(authorsOriginalOrder.get(1)); + + moveTraditionalPageOneAuthorTest(1, 4, expectedOrder); + } + + /** + * This test will replace an author (dc.contributor.author) within a workspace publication's "traditionalpageone" + * section at position 0 using a PATCH request and verify the order and value of the authors within the section. + * This test uses only plain text authors + * @throws Exception + */ + @Test + public void replaceTraditionalPagePlainTextOneAuthorZeroTest() throws Exception { + initPlainTextPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(replacedAuthor); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + replaceTraditionalPageOneAuthorTest(0, expectedOrder); + } + + /** + * This test will replace an author (dc.contributor.author) within a workspace publication's "traditionalpageone" + * section at position 2 using a PATCH request and verify the order and value of the authors within the section. + * This test uses only plain text authors + * @throws Exception + */ + @Test + public void replaceTraditionalPagePlainTextOneAuthorTwoTest() throws Exception { + initPlainTextPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(replacedAuthor); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + replaceTraditionalPageOneAuthorTest(2, expectedOrder); + } + + /** + * This test will replace an author (dc.contributor.author) within a workspace publication's "traditionalpageone" + * section at position 3 using a PATCH request and verify the order and value of the authors within the section. + * This test uses only plain text authors + * @throws Exception + */ + @Test + public void replaceTraditionalPageOnePlainTextAuthorThreeTest() throws Exception { + initPlainTextPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(replacedAuthor); + expectedOrder.add(authorsOriginalOrder.get(4)); + + replaceTraditionalPageOneAuthorTest(3, expectedOrder); + } + + + /** + * This test will add an author (dc.contributor.author) within a workspace publication's "traditionalpageone" + * section at position 0 using a PATCH request and verify the place of the new author and the order of the + * authors within the section. + * This test uses only plain text authors + * Original Order: 0,1,2,3,4 + * Expected Order: +,0,1,2,3,4 (with + being the new author) + */ + @Test + public void addAuthorOnTraditionalPageOnePlainTextPlaceZeroTest() throws Exception { + initPlainTextPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(addedAuthor); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + addTraditionalPageOneAuthorTest("0", expectedOrder); + + } + + /** + * This test will add an author (dc.contributor.author) within a workspace publication's "traditionalpageone" + * section at position 1 using a PATCH request and verify the place of the new author and the order of the + * authors within the section. + * This test uses only plain text authors + * Original Order: 0,1,2,3,4 + * Expected Order: 0,+,1,2,3,4 (with + being the new author) + */ + @Test + public void addAuthorOnTraditionalPageOnePlainTextPlaceOneTest() throws Exception { + initPlainTextPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(addedAuthor); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + addTraditionalPageOneAuthorTest("1", expectedOrder); + + } + + /** + * This test will add an author (dc.contributor.author) within a workspace publication's "traditionalpageone" + * section at position 2 using a PATCH request and verify the place of the new author and the order of the + * authors within the section. + * This test uses only plain text authors + * Original Order: 0,1,2,3,4 + * Expected Order: 0,1,+,2,3,4 (with + being the new author) + */ + @Test + public void addAuthorOnTraditionalPageOnePlainTextPlaceTwoTest() throws Exception { + initPlainTextPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(addedAuthor); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + addTraditionalPageOneAuthorTest("2", expectedOrder); + + } + + /** + * This test will add an author (dc.contributor.author) within a workspace publication's "traditionalpageone" + * section at position 3 using a PATCH request and verify the place of the new author and the order of the + * authors within the section. + * This test uses only plain text authors + * Original Order: 0,1,2,3,4 + * Expected Order: 0,1,2,+,3,4 (with + being the new author) + */ + @Test + public void addAuthorOnTraditionalPageOnePlainTextPlaceThreeTest() throws Exception { + initPlainTextPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(addedAuthor); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + addTraditionalPageOneAuthorTest("3", expectedOrder); + + } + + /** + * This test will add an author (dc.contributor.author) within a workspace publication's "traditionalpageone" + * section at position 4 using a PATCH request and verify the place of the new author and the order of the + * authors within the section. + * This test uses only plain text authors + * Original Order: 0,1,2,3,4 + * Expected Order: 0,1,2,3,+,4 (with + being the new author) + */ + @Test + public void addAuthorOnTraditionalPageOnePlainTextPlaceFourTest() throws Exception { + initPlainTextPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(addedAuthor); + expectedOrder.add(authorsOriginalOrder.get(4)); + + addTraditionalPageOneAuthorTest("4", expectedOrder); + + } + + /** + * This test will add an author (dc.contributor.author) within a workspace publication's "traditionalpageone" + * section at position 0 using a PATCH request and verify the place of the new author and the order of the + * authors within the section. + * This test uses only plain text authors + * Original Order: 0,1,2,3,4 + * Expected Order: +,0,1,2,3,4 (with + being the new author) + */ + @Test + public void addAuthorOnTraditionalPageOneLastPlainTextPlaceTest() throws Exception { + initPlainTextPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(4)); + expectedOrder.add(addedAuthor); + + addTraditionalPageOneAuthorTest("-", expectedOrder); + + } + + /** + * This test will remove the author (dc.contributor.author) from a workspace publication's "traditionalpageone" + * section at position 0 using a PATCH request and verify the order of the remaining authors within the section. + * This test uses only plain text authors + * Original Order: 0,1,2,3,4 + * Expected Order: 1,2,3,4 + */ + @Test + public void removeAuthorOnTraditionalPagePlainTextFromPlaceZeroTest() throws Exception { + initPlainTextPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + removeTraditionalPageOneAuthorTest(0, expectedOrder); + } + /** + * This test will remove the author (dc.contributor.author) from a workspace publication's "traditionalpageone" + * section at position 1 using a PATCH request and verify the order of the remaining authors within the section. + * This test uses only plain text authors + * Original Order: 0,1,2,3,4 + * Expected Order: 0,2,3,4 + */ + @Test + public void removeAuthorOnTraditionalPagePlainTextFromPlaceOneTest() throws Exception { + initPlainTextPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + // The author at the first place is linked through a relationship and cannot be deleted through a PATCH request + removeTraditionalPageOneAuthorTest(1, expectedOrder); + } + /** + * This test will remove the author (dc.contributor.author) from a workspace publication's "traditionalpageone" + * section at position 2 using a PATCH request and verify the order of the remaining authors within the section. + * This test uses only plain text authors + * Original Order: 0,1,2,3,4 + * Expected Order: 0,1,3,4 + */ + @Test + public void removeAuthorOnTraditionalPagePlainTextFromPlaceTwoTest() throws Exception { + initPlainTextPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(3)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + removeTraditionalPageOneAuthorTest(2, expectedOrder); + } + /** + * This test will remove the author (dc.contributor.author) from a workspace publication's "traditionalpageone" + * section at position 3 using a PATCH request and verify the order of the remaining authors within the section. + * This test uses only plain text authors + * Original Order: 0,1,2,3,4 + * Expected Order: 0,1,2,4 + */ + @Test + public void removeAuthorOnTraditionalPagePlainTextFromPlaceThreeTest() throws Exception { + initPlainTextPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(4)); + + removeTraditionalPageOneAuthorTest(3, expectedOrder); + } + /** + * This test will remove the author (dc.contributor.author) from a workspace publication's "traditionalpageone" + * section at position 4 using a PATCH request and verify the order of the remaining authors within the section. + * This test uses only plain text authors + * Original Order: 0,1,2,3,4 + * Expected Order: 0,1,2,3 + */ + @Test + public void removeAuthorOnTraditionalPagePlainTextFromPlaceFourTest() throws Exception { + initPlainTextPublicationWorkspace(); + + List expectedOrder = new ArrayList<>(); + expectedOrder.add(authorsOriginalOrder.get(0)); + expectedOrder.add(authorsOriginalOrder.get(1)); + expectedOrder.add(authorsOriginalOrder.get(2)); + expectedOrder.add(authorsOriginalOrder.get(3)); + + // The author at the fourth place is linked through a relationship and cannot be deleted through a PATCH request + removeTraditionalPageOneAuthorTest(4, expectedOrder); + } + + /** * This test will remove all authors (dc.contributor.author) that are not linked through a relationship from a * workspace publication's "traditionalpageone" section using a PATCH request and verify that the only remaining From 483ec64ce024b2db6c29db118607446619db3bc2 Mon Sep 17 00:00:00 2001 From: Kevin Van de Velde Date: Fri, 7 Aug 2020 13:47:20 +0200 Subject: [PATCH 53/96] [Issue: 2821] Metadata-export adding file name extension --- .../src/main/java/org/dspace/app/bulkedit/MetadataExport.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExport.java b/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExport.java index a91b83e6d9..195a10e2e2 100644 --- a/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExport.java +++ b/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExport.java @@ -91,6 +91,6 @@ public class MetadataExport extends DSpaceRunnable Date: Fri, 7 Aug 2020 16:52:22 +0200 Subject: [PATCH 54/96] 72428: Update java docs --- .../content/service/DSpaceObjectService.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/content/service/DSpaceObjectService.java b/dspace-api/src/main/java/org/dspace/content/service/DSpaceObjectService.java index e1eeca062e..cf46ad909f 100644 --- a/dspace-api/src/main/java/org/dspace/content/service/DSpaceObjectService.java +++ b/dspace-api/src/main/java/org/dspace/content/service/DSpaceObjectService.java @@ -200,7 +200,7 @@ public interface DSpaceObjectService { * and the ISO3166 country code. null means the * value has no language (for example, a date). * @param values the values to add. - * @return the list of MetadataValues of the object including the newly added values + * @return the list of MetadataValues added ot the object * @throws SQLException if database error */ public List addMetadata(Context context, T dso, String schema, String element, String qualifier, @@ -224,7 +224,7 @@ public interface DSpaceObjectService { * @param values the values to add. * @param authorities the external authority key for this value (or null) * @param confidences the authority confidence (default 0) - * @return the list of MetadataValues of the object including the newly added values + * @return the list of MetadataValues added ot the object * @throws SQLException if database error */ public List addMetadata(Context context, T dso, String schema, String element, String qualifier, @@ -245,7 +245,7 @@ public interface DSpaceObjectService { * @param values the values to add. * @param authorities the external authority key for this value (or null) * @param confidences the authority confidence (default 0) - * @return the list of MetadataValues of the object including the newly added values + * @return the list of MetadataValues added ot the object * @throws SQLException if database error */ public List addMetadata(Context context, T dso, MetadataField metadataField, String lang, @@ -264,7 +264,7 @@ public interface DSpaceObjectService { * @param value the value to add. * @param authority the external authority key for this value (or null) * @param confidence the authority confidence (default 0) - * @return the first MetadataValue of the DSpaceObject + * @return the MetadataValue added ot the object * @throws SQLException */ public MetadataValue addMetadata(Context context, T dso, MetadataField metadataField, String language, @@ -281,7 +281,7 @@ public interface DSpaceObjectService { * and the ISO3166 country code. null means the * value has no language (for example, a date). * @param value the value to add. - * @return the first MetadataValue of the DSpaceObject + * @return the MetadataValue added ot the object * @throws SQLException if database error */ public MetadataValue addMetadata(Context context, T dso, MetadataField metadataField, String language, String value) @@ -298,7 +298,7 @@ public interface DSpaceObjectService { * and the ISO3166 country code. null means the * value has no language (for example, a date). * @param values the values to add. - * @return the list of MetadataValues of the object including the newly added values + * @return the list of MetadataValues added ot the object * @throws SQLException if database error */ public List addMetadata(Context context, T dso, MetadataField metadataField, String language, @@ -319,7 +319,7 @@ public interface DSpaceObjectService { * and the ISO3166 country code. null means the * value has no language (for example, a date). * @param value the value to add. - * @return the first MetadataValue of the DSpaceObject + * @return the MetadataValue added ot the object * @throws SQLException if database error */ public MetadataValue addMetadata(Context context, T dso, String schema, String element, String qualifier, @@ -342,7 +342,7 @@ public interface DSpaceObjectService { * @param value the value to add. * @param authority the external authority key for this value (or null) * @param confidence the authority confidence (default 0) - * @return the first MetadataValue of the DSpaceObject + * @return the MetadataValue added ot the object * @throws SQLException if database error */ public MetadataValue addMetadata(Context context, T dso, String schema, String element, String qualifier, From b4e64e5bf373ee2be6cb304f88b0177ba131c4b1 Mon Sep 17 00:00:00 2001 From: Marie Verdonck Date: Fri, 7 Aug 2020 17:10:32 +0200 Subject: [PATCH 55/96] 71738: Sitemap scheduler with annotations --- .../java/org/dspace/app/rest/Application.java | 11 ++++++++++ dspace/config/dspace.cfg | 3 ++- dspace/config/spring/api/task-scheduler.xml | 20 ------------------- 3 files changed, 13 insertions(+), 21 deletions(-) delete mode 100644 dspace/config/spring/api/task-scheduler.xml diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/Application.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/Application.java index 73f1cbfd80..a2ea0d1c3c 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/Application.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/Application.java @@ -7,6 +7,8 @@ */ package org.dspace.app.rest; +import java.io.IOException; +import java.sql.SQLException; import java.util.List; import javax.servlet.Filter; @@ -16,6 +18,7 @@ import org.dspace.app.rest.parameter.resolver.SearchFilterResolver; import org.dspace.app.rest.utils.ApplicationConfig; import org.dspace.app.rest.utils.DSpaceConfigurationInitializer; import org.dspace.app.rest.utils.DSpaceKernelInitializer; +import org.dspace.app.sitemap.GenerateSitemaps; import org.dspace.app.util.DSpaceContextListener; import org.dspace.utils.servlet.DSpaceWebappServletFilter; import org.slf4j.Logger; @@ -28,6 +31,8 @@ import org.springframework.context.annotation.Bean; import org.springframework.core.annotation.Order; import org.springframework.hateoas.server.LinkRelationProvider; import org.springframework.lang.NonNull; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.web.context.request.RequestContextListener; import org.springframework.web.cors.CorsConfiguration; import org.springframework.web.method.support.HandlerMethodArgumentResolver; @@ -49,6 +54,7 @@ import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; * @author Tim Donohue */ @SpringBootApplication +@EnableScheduling public class Application extends SpringBootServletInitializer { private static final Logger log = LoggerFactory.getLogger(Application.class); @@ -56,6 +62,11 @@ public class Application extends SpringBootServletInitializer { @Autowired private ApplicationConfig configuration; + @Scheduled(cron = "${sitemap.cron:-}") + public void generateSitemap() throws IOException, SQLException { + GenerateSitemaps.generateSitemapsScheduled(); + } + /** * Override the default SpringBootServletInitializer.configure() method, * passing it this Application class. diff --git a/dspace/config/dspace.cfg b/dspace/config/dspace.cfg index df21333945..2e3addd2e1 100644 --- a/dspace/config/dspace.cfg +++ b/dspace/config/dspace.cfg @@ -1447,8 +1447,9 @@ sitemap.engineurls = http://www.google.com/webmasters/sitemaps/ping?sitemap= # Define cron for how frequently the sitemap should refresh. # Defaults to running daily at 1:15am +# Remove (comment) out this config to disable the sitemap scheduler # Syntax is defined at https://www.quartz-scheduler.org/api/2.3.0/org/quartz/CronTrigger.html -sitemap.cron = 0 * * * * ? +sitemap.cron = 0 15 1 * * ? ##### SHERPA/Romeo Integration Settings #### # the SHERPA/RoMEO endpoint diff --git a/dspace/config/spring/api/task-scheduler.xml b/dspace/config/spring/api/task-scheduler.xml deleted file mode 100644 index da6d64730a..0000000000 --- a/dspace/config/spring/api/task-scheduler.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - - From 0f90b8a8713d811a0a6037367e1d746250e563f6 Mon Sep 17 00:00:00 2001 From: Yana De Pauw Date: Mon, 10 Aug 2020 09:31:37 +0200 Subject: [PATCH 56/96] fix typo in javadocs --- .../org/dspace/content/service/DSpaceObjectService.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/content/service/DSpaceObjectService.java b/dspace-api/src/main/java/org/dspace/content/service/DSpaceObjectService.java index cf46ad909f..ff44713b38 100644 --- a/dspace-api/src/main/java/org/dspace/content/service/DSpaceObjectService.java +++ b/dspace-api/src/main/java/org/dspace/content/service/DSpaceObjectService.java @@ -200,7 +200,7 @@ public interface DSpaceObjectService { * and the ISO3166 country code. null means the * value has no language (for example, a date). * @param values the values to add. - * @return the list of MetadataValues added ot the object + * @return the list of MetadataValues added to the object * @throws SQLException if database error */ public List addMetadata(Context context, T dso, String schema, String element, String qualifier, @@ -224,7 +224,7 @@ public interface DSpaceObjectService { * @param values the values to add. * @param authorities the external authority key for this value (or null) * @param confidences the authority confidence (default 0) - * @return the list of MetadataValues added ot the object + * @return the list of MetadataValues added to the object * @throws SQLException if database error */ public List addMetadata(Context context, T dso, String schema, String element, String qualifier, @@ -245,7 +245,7 @@ public interface DSpaceObjectService { * @param values the values to add. * @param authorities the external authority key for this value (or null) * @param confidences the authority confidence (default 0) - * @return the list of MetadataValues added ot the object + * @return the list of MetadataValues added to the object * @throws SQLException if database error */ public List addMetadata(Context context, T dso, MetadataField metadataField, String lang, @@ -298,7 +298,7 @@ public interface DSpaceObjectService { * and the ISO3166 country code. null means the * value has no language (for example, a date). * @param values the values to add. - * @return the list of MetadataValues added ot the object + * @return the list of MetadataValues added to the object * @throws SQLException if database error */ public List addMetadata(Context context, T dso, MetadataField metadataField, String language, From 0d5f5174c0a5088cccf553d66a04c01975b3fa1c Mon Sep 17 00:00:00 2001 From: Yana De Pauw Date: Mon, 10 Aug 2020 10:51:47 +0200 Subject: [PATCH 57/96] fix build issue --- .../test/java/org/dspace/app/rest/PatchMetadataIT.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java index 1e16eefb27..a9178defc2 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/PatchMetadataIT.java @@ -26,11 +26,6 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicReference; -import org.dspace.app.rest.builder.CollectionBuilder; -import org.dspace.app.rest.builder.CommunityBuilder; -import org.dspace.app.rest.builder.ItemBuilder; -import org.dspace.app.rest.builder.RelationshipBuilder; -import org.dspace.app.rest.builder.WorkspaceItemBuilder; import org.dspace.app.rest.matcher.MetadataMatcher; import org.dspace.app.rest.model.MetadataValueRest; import org.dspace.app.rest.model.patch.AddOperation; @@ -39,6 +34,11 @@ import org.dspace.app.rest.model.patch.Operation; import org.dspace.app.rest.model.patch.RemoveOperation; import org.dspace.app.rest.model.patch.ReplaceOperation; import org.dspace.app.rest.test.AbstractEntityIntegrationTest; +import org.dspace.builder.CollectionBuilder; +import org.dspace.builder.CommunityBuilder; +import org.dspace.builder.ItemBuilder; +import org.dspace.builder.RelationshipBuilder; +import org.dspace.builder.WorkspaceItemBuilder; import org.dspace.content.Collection; import org.dspace.content.Community; import org.dspace.content.Item; From 9fa1b78e2efc9847397bb64122d416fce8e279c7 Mon Sep 17 00:00:00 2001 From: Yana De Pauw Date: Mon, 10 Aug 2020 15:51:04 +0200 Subject: [PATCH 58/96] 72450: AdministratorOfFeature: fixes --- .../builder/AbstractDSpaceObjectBuilder.java | 27 ++ .../java/org/dspace/builder/ItemBuilder.java | 14 + .../impl/AdministratorOfFeature.java | 9 +- .../authorization/AdministratorFeatureIT.java | 415 ++++++++++++------ 4 files changed, 341 insertions(+), 124 deletions(-) diff --git a/dspace-api/src/test/java/org/dspace/builder/AbstractDSpaceObjectBuilder.java b/dspace-api/src/test/java/org/dspace/builder/AbstractDSpaceObjectBuilder.java index df92d0ea4b..075a533273 100644 --- a/dspace-api/src/test/java/org/dspace/builder/AbstractDSpaceObjectBuilder.java +++ b/dspace-api/src/test/java/org/dspace/builder/AbstractDSpaceObjectBuilder.java @@ -146,6 +146,33 @@ public abstract class AbstractDSpaceObjectBuilder } return (B) this; } + /** + * Support method to grant the {@link Constants#READ} permission over an object only to a specific group. Any other + * READ permissions will be removed + * + * @param dso + * the DSpaceObject on which grant the permission + * @param eperson + * the eperson that will be granted of the permission + * @return the builder properly configured to build the object with the additional admin permission + */ + protected > B setAdminPermission(DSpaceObject dso, EPerson eperson, + Date startDate) { + try { + + ResourcePolicy rp = authorizeService.createOrModifyPolicy(null, context, null, null, + eperson, startDate, Constants.ADMIN, + "Integration Test", dso); + if (rp != null) { + log.info("Updating resource policy with REMOVE for eperson: " + eperson.getEmail()); + resourcePolicyService.update(context, rp); + } + } catch (Exception e) { + return handleException(e); + } + return (B) this; + + } /** * Support method to grant {@link Constants#REMOVE} permission to a specific eperson diff --git a/dspace-api/src/test/java/org/dspace/builder/ItemBuilder.java b/dspace-api/src/test/java/org/dspace/builder/ItemBuilder.java index e4f3a0e29d..ca2699c283 100644 --- a/dspace-api/src/test/java/org/dspace/builder/ItemBuilder.java +++ b/dspace-api/src/test/java/org/dspace/builder/ItemBuilder.java @@ -19,6 +19,7 @@ import org.dspace.content.MetadataSchemaEnum; import org.dspace.content.WorkspaceItem; import org.dspace.content.service.DSpaceObjectService; import org.dspace.core.Context; +import org.dspace.eperson.EPerson; import org.dspace.eperson.Group; /** @@ -126,6 +127,19 @@ public class ItemBuilder extends AbstractDSpaceObjectBuilder { return this; } + /** + * Create an admin group for the collection with the specified members + * + * @param members epersons to add to the admin group + * @return this builder + * @throws SQLException + * @throws AuthorizeException + */ + public ItemBuilder withAdminUser(EPerson ePerson) throws SQLException, AuthorizeException { + return setAdminPermission(item, ePerson, null); + } + + @Override public Item build() { try { diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/authorization/impl/AdministratorOfFeature.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/authorization/impl/AdministratorOfFeature.java index 5ce4977b5a..6cfee12751 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/authorization/impl/AdministratorOfFeature.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/authorization/impl/AdministratorOfFeature.java @@ -14,11 +14,13 @@ import org.dspace.app.rest.authorization.AuthorizationFeatureDocumentation; import org.dspace.app.rest.model.BaseObjectRest; import org.dspace.app.rest.model.CollectionRest; import org.dspace.app.rest.model.CommunityRest; +import org.dspace.app.rest.model.ItemRest; import org.dspace.app.rest.model.SiteRest; import org.dspace.app.rest.utils.Utils; import org.dspace.authorize.service.AuthorizeService; import org.dspace.content.Collection; import org.dspace.content.Community; +import org.dspace.content.Item; import org.dspace.core.Context; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -53,6 +55,10 @@ public class AdministratorOfFeature implements AuthorizationFeature { Collection collection = (Collection) utils.getDSpaceAPIObjectFromRest(context, object); return authService.isAdmin(context, collection); } + if (object instanceof ItemRest) { + Item item = (Item) utils.getDSpaceAPIObjectFromRest(context, object); + return authService.isAdmin(context, item); + } } return authService.isAdmin(context); } @@ -62,7 +68,8 @@ public class AdministratorOfFeature implements AuthorizationFeature { return new String[]{ SiteRest.CATEGORY + "." + SiteRest.NAME, CommunityRest.CATEGORY + "." + CommunityRest.NAME, - CollectionRest.CATEGORY + "." + CollectionRest.NAME + CollectionRest.CATEGORY + "." + CollectionRest.NAME, + ItemRest.CATEGORY + "." + ItemRest.NAME }; } } \ No newline at end of file diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/authorization/AdministratorFeatureIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/authorization/AdministratorFeatureIT.java index c8e35cb10b..6124e2cac7 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/authorization/AdministratorFeatureIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/authorization/AdministratorFeatureIT.java @@ -11,22 +11,29 @@ import static org.springframework.test.web.servlet.request.MockMvcRequestBuilder import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import java.sql.SQLException; + import org.dspace.app.rest.authorization.impl.AdministratorOfFeature; import org.dspace.app.rest.converter.CollectionConverter; import org.dspace.app.rest.converter.CommunityConverter; +import org.dspace.app.rest.converter.ItemConverter; import org.dspace.app.rest.converter.SiteConverter; import org.dspace.app.rest.matcher.AuthorizationMatcher; import org.dspace.app.rest.model.CollectionRest; import org.dspace.app.rest.model.CommunityRest; +import org.dspace.app.rest.model.ItemRest; import org.dspace.app.rest.model.SiteRest; import org.dspace.app.rest.projection.DefaultProjection; import org.dspace.app.rest.test.AbstractControllerIntegrationTest; +import org.dspace.authorize.AuthorizeException; import org.dspace.authorize.service.AuthorizeService; import org.dspace.builder.CollectionBuilder; import org.dspace.builder.CommunityBuilder; import org.dspace.builder.EPersonBuilder; +import org.dspace.builder.ItemBuilder; import org.dspace.content.Collection; import org.dspace.content.Community; +import org.dspace.content.Item; import org.dspace.content.Site; import org.dspace.content.factory.ContentServiceFactory; import org.dspace.content.service.CommunityService; @@ -55,6 +62,8 @@ public class AdministratorFeatureIT extends AbstractControllerIntegrationTest { @Autowired CommunityService communityService; @Autowired + private ItemConverter itemConverter; + @Autowired private CommunityConverter communityConverter; @Autowired private CollectionConverter collectionConverter; @@ -63,6 +72,22 @@ public class AdministratorFeatureIT extends AbstractControllerIntegrationTest { private SiteService siteService; + private EPerson adminComA; + private EPerson adminComB; + private EPerson adminColA; + private EPerson adminColB; + private EPerson adminItemA; + private EPerson adminItemB; + + private Community communityA; + private Community subCommunityOfA; + private Community communityB; + private Collection collectionA; + private Collection collectionB; + private Item itemInCollectionA; + private Item itemInCollectionB; + + /** * this hold a reference to the test feature {@link AdministratorOfFeature} */ @@ -74,201 +99,345 @@ public class AdministratorFeatureIT extends AbstractControllerIntegrationTest { super.setUp(); siteService = ContentServiceFactory.getInstance().getSiteService(); administratorFeature = authorizationFeatureService.find(AdministratorOfFeature.NAME); + initAdminsAndObjects(); + } + + private void initAdminsAndObjects() throws SQLException, AuthorizeException { + context.turnOffAuthorisationSystem(); + + + adminComA = EPersonBuilder.createEPerson(context) + .withEmail("adminComA@example.com") + .withPassword(password) + .build(); + + adminComB = EPersonBuilder.createEPerson(context) + .withEmail("adminComB@example.com") + .withPassword(password) + .build(); + + adminColA = EPersonBuilder.createEPerson(context) + .withEmail("adminColA@example.com") + .withPassword(password) + .build(); + + adminColB = EPersonBuilder.createEPerson(context) + .withEmail("adminColB@example.com") + .withPassword(password) + .build(); + + adminItemA = EPersonBuilder.createEPerson(context) + .withEmail("adminItemA@example.com") + .withPassword(password) + .build(); + + adminItemB = EPersonBuilder.createEPerson(context) + .withEmail("adminItemB@example.com") + .withPassword(password) + .build(); + + communityA = CommunityBuilder.createCommunity(context) + .withName("Community A") + .withAdminGroup(adminComA) + .build(); + + subCommunityOfA = CommunityBuilder.createSubCommunity(context, communityA) + .withName("Sub Community of CommunityA") + .build(); + + communityB = CommunityBuilder.createCommunity(context) + .withName("Community B") + .withAdminGroup(adminComB) + .build(); + + collectionA = CollectionBuilder.createCollection(context, subCommunityOfA) + .withName("Collection A") + .withAdminGroup(adminColA) + .build(); + + collectionB = CollectionBuilder.createCollection(context, communityB) + .withName("Collection B") + .withAdminGroup(adminColB) + .build(); + + itemInCollectionA = ItemBuilder.createItem(context, collectionA) + .withTitle("Item in Collection A") + .withAdminUser(adminItemA) + .build(); + + itemInCollectionB = ItemBuilder.createItem(context, collectionB) + .withTitle("Item in Collection B") + .withAdminUser(adminItemB) + .build(); + + context.restoreAuthSystemState(); } @Test public void communityWithAdministratorFeatureTest() throws Exception { - context.turnOffAuthorisationSystem(); - EPerson adminComA = EPersonBuilder.createEPerson(context) - .withEmail("adminComA@example.com") - .withPassword(password) - .build(); - - EPerson adminComB = EPersonBuilder.createEPerson(context) - .withEmail("adminComB@example.com") - .withPassword(password) - .build(); - - Community communityA = CommunityBuilder.createCommunity(context) - .withName("Community A") - .withAdminGroup(adminComA) - .build(); - - Community subCommunityOfA = CommunityBuilder.createSubCommunity(context, communityA) - .withName("Sub Community of CommunityA") - .build(); - - Collection collectionOfSubComm = CollectionBuilder.createCollection(context, subCommunityOfA) - .withName("Collection of subCommunity") - .build(); - - Community communityB = CommunityBuilder.createCommunity(context) - .withName("Community B") - .withAdminGroup(adminComB) - .build(); - - context.restoreAuthSystemState(); - CommunityRest communityRestA = communityConverter.convert(communityA, DefaultProjection.DEFAULT); - CommunityRest SubCommunityOfArest = communityConverter.convert(subCommunityOfA, DefaultProjection.DEFAULT); - CollectionRest collectionRestOfSubComm = collectionConverter.convert(collectionOfSubComm, - DefaultProjection.DEFAULT); + CommunityRest communityRestB = communityConverter.convert(communityB, DefaultProjection.DEFAULT); + CommunityRest SubCommunityOfARest = communityConverter.convert(subCommunityOfA, DefaultProjection.DEFAULT); // tokens String tokenAdminComA = getAuthToken(adminComA.getEmail(), password); String tokenAdminComB = getAuthToken(adminComB.getEmail(), password); + String tokenAdmin = getAuthToken(admin.getEmail(), password); // define authorizations that we know must exists - Authorization authAdminCommunityA = new Authorization(adminComA, administratorFeature, communityRestA); - Authorization authAdminSubCommunityOfA = new Authorization(adminComA, administratorFeature,SubCommunityOfArest); - Authorization authAdminAColl = new Authorization(adminComA, administratorFeature, collectionRestOfSubComm); + Authorization authAdminSiteComA = new Authorization(admin, administratorFeature, communityRestA); + Authorization authAdminComAComA = new Authorization(adminComA, administratorFeature, communityRestA); + Authorization authAdminComASubComA = new Authorization(adminComA, administratorFeature, SubCommunityOfARest); + Authorization authAdminComBComB = new Authorization(adminComB, administratorFeature, communityRestB); + // define authorizations that we know not exists - Authorization authAdminBColl = new Authorization(adminComB, administratorFeature, collectionRestOfSubComm); - Authorization authAdminBCommunityA = new Authorization(adminComB, administratorFeature, communityRestA); + Authorization authAdminComBComA = new Authorization(adminComB, administratorFeature, communityRestA); + Authorization authAdminComBSubComA = new Authorization(adminComB, administratorFeature, SubCommunityOfARest); + Authorization authAdminColAComA = new Authorization(adminColA, administratorFeature, communityRestA); + Authorization authAdminItemAComA = new Authorization(adminItemA, administratorFeature, communityRestA); + Authorization authEPersonComA = new Authorization(eperson, administratorFeature, communityRestA); + Authorization authAnonymousComA = new Authorization(null, administratorFeature, communityRestA); - getClient(tokenAdminComA).perform(get("/api/authz/authorizations/" + authAdminCommunityA.getID())) + + + getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authAdminSiteComA.getID())) .andExpect(status().isOk()) - .andExpect(jsonPath("$", Matchers.is(AuthorizationMatcher.matchAuthorization(authAdminCommunityA)))); + .andExpect(jsonPath("$", Matchers.is(AuthorizationMatcher.matchAuthorization(authAdminSiteComA)))); - getClient(tokenAdminComA).perform(get("/api/authz/authorizations/" + authAdminSubCommunityOfA.getID())) + getClient(tokenAdminComA).perform(get("/api/authz/authorizations/" + authAdminComAComA.getID())) .andExpect(status().isOk()) .andExpect(jsonPath("$", Matchers.is(AuthorizationMatcher - .matchAuthorization(authAdminSubCommunityOfA)))); + .matchAuthorization(authAdminComAComA)))); - getClient(tokenAdminComA).perform(get("/api/authz/authorizations/" + authAdminAColl.getID())) + getClient(tokenAdminComA).perform(get("/api/authz/authorizations/" + authAdminComASubComA.getID())) .andExpect(status().isOk()) - .andExpect(jsonPath("$", Matchers.is(AuthorizationMatcher.matchAuthorization(authAdminAColl)))); + .andExpect(jsonPath("$", Matchers.is(AuthorizationMatcher.matchAuthorization(authAdminComASubComA)))); - getClient(tokenAdminComB).perform(get("/api/authz/authorizations/" + authAdminBCommunityA.getID())) + getClient(tokenAdminComB).perform(get("/api/authz/authorizations/" + authAdminComBComB.getID())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", Matchers.is(AuthorizationMatcher.matchAuthorization(authAdminComBComB)))); + + getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authAdminComBComA.getID())) .andExpect(status().isNotFound()); - getClient(tokenAdminComB).perform(get("/api/authz/authorizations/" + authAdminBColl.getID())) + getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authAdminComBSubComA.getID())) .andExpect(status().isNotFound()); + getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authAdminColAComA.getID())) + .andExpect(status().isNotFound()); + getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authAdminItemAComA.getID())) + .andExpect(status().isNotFound()); + getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authEPersonComA.getID())) + .andExpect(status().isNotFound()); + getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authAnonymousComA.getID())) + .andExpect(status().isNotFound()); + + } @Test public void collectionWithAdministratorFeatureTest() throws Exception { - context.turnOffAuthorisationSystem(); - - EPerson adminColA = EPersonBuilder.createEPerson(context) - .withEmail("adminColA@example.com") - .withPassword(password) - .build(); - - EPerson adminColB = EPersonBuilder.createEPerson(context) - .withEmail("adminColB@example.com") - .withPassword(password) - .build(); - - Community parentCommunity = CommunityBuilder.createCommunity(context) - .withName("Parent Community") - .build(); - - Collection collectionA = CollectionBuilder.createCollection(context, parentCommunity) - .withName("Collection A") - .withAdminGroup(adminColA) - .build(); - - Collection collectionB = CollectionBuilder.createCollection(context, parentCommunity) - .withName("Collection B") - .withAdminGroup(adminColB) - .build(); - - context.restoreAuthSystemState(); - CollectionRest collectionRestA = collectionConverter.convert(collectionA, DefaultProjection.DEFAULT); CollectionRest collectionRestB = collectionConverter.convert(collectionB, DefaultProjection.DEFAULT); String tokenAdminColA = getAuthToken(adminColA.getEmail(), password); String tokenAdminColB = getAuthToken(adminColB.getEmail(), password); + String tokenAdminComA = getAuthToken(adminComA.getEmail(), password); + String tokenAdminComB = getAuthToken(adminComB.getEmail(), password); + String tokenAdmin = getAuthToken(admin.getEmail(), password); // define authorizations that we know must exists - Authorization authAdminCollectionA = new Authorization(adminColA, administratorFeature, collectionRestA); - Authorization authAdminCollectionB = new Authorization(adminColB, administratorFeature, collectionRestB); + + Authorization authAdminSiteColA = new Authorization(admin, administratorFeature, collectionRestA); + Authorization authAdminComAColA = new Authorization(adminComA, administratorFeature, collectionRestA); + Authorization authAdminColAColA = new Authorization(adminColA, administratorFeature, collectionRestA); + + Authorization authAdminSiteColB = new Authorization(admin, administratorFeature, collectionRestB); + Authorization authAdminComBColB = new Authorization(adminComB, administratorFeature, collectionRestB); + Authorization authAdminColBColB = new Authorization(adminColB, administratorFeature, collectionRestB); // define authorization that we know not exists - Authorization authAdminBcollectionA = new Authorization(adminColB, administratorFeature, collectionRestA); + Authorization authAdminColBColA = new Authorization(adminColB, administratorFeature, collectionRestA); + Authorization authAdminComBColA = new Authorization(adminComB, administratorFeature, collectionRestA); + Authorization authAdminItemAColA = new Authorization(adminItemA, administratorFeature, collectionRestA); + Authorization authEPersonColA = new Authorization(eperson, administratorFeature, collectionRestA); + Authorization authAnonymousColA = new Authorization(null, administratorFeature, collectionRestA); - getClient(tokenAdminColA).perform(get("/api/authz/authorizations/" + authAdminCollectionA.getID())) - .andExpect(status().isOk()) - .andExpect(jsonPath("$", Matchers.is(AuthorizationMatcher.matchAuthorization(authAdminCollectionA)))); - getClient(tokenAdminColB).perform(get("/api/authz/authorizations/" + authAdminCollectionB.getID())) - .andExpect(status().isOk()) - .andExpect(jsonPath("$", Matchers.is(AuthorizationMatcher.matchAuthorization(authAdminCollectionB)))); - getClient(tokenAdminColB).perform(get("/api/authz/authorizations/" + authAdminBcollectionA.getID())) - .andExpect(status().isNotFound()); + getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authAdminSiteColA.getID())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", Matchers.is( + AuthorizationMatcher.matchAuthorization(authAdminSiteColA)))); + + getClient(tokenAdminComA).perform(get("/api/authz/authorizations/" + authAdminComAColA.getID())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", Matchers.is( + AuthorizationMatcher.matchAuthorization(authAdminComAColA)))); + + getClient(tokenAdminColA).perform(get("/api/authz/authorizations/" + authAdminColAColA.getID())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", Matchers.is( + AuthorizationMatcher.matchAuthorization(authAdminColAColA)))); + + + getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authAdminSiteColB.getID())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", Matchers.is( + AuthorizationMatcher.matchAuthorization(authAdminSiteColB)))); + + getClient(tokenAdminComB).perform(get("/api/authz/authorizations/" + authAdminComBColB.getID())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", Matchers.is( + AuthorizationMatcher.matchAuthorization(authAdminComBColB)))); + + getClient(tokenAdminColB).perform(get("/api/authz/authorizations/" + authAdminColBColB.getID())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", Matchers.is( + AuthorizationMatcher.matchAuthorization(authAdminColBColB)))); + + + getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authAdminColBColA.getID())) + .andExpect(status().isNotFound()); + getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authAdminComBColA.getID())) + .andExpect(status().isNotFound()); + getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authAdminItemAColA.getID())) + .andExpect(status().isNotFound()); + getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authEPersonColA.getID())) + .andExpect(status().isNotFound()); + getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authAnonymousColA.getID())) + .andExpect(status().isNotFound()); } @Test public void siteWithAdministratorFeatureTest() throws Exception { - context.turnOffAuthorisationSystem(); - - Community parentCommunity = CommunityBuilder.createCommunity(context) - .withName("Test Parent Community") - .build(); - - Collection collection = CollectionBuilder.createCollection(context, parentCommunity) - .withName("Test Collection") - .build(); - - context.restoreAuthSystemState(); Site site = siteService.findSite(context); SiteRest siteRest = siteConverter.convert(site, DefaultProjection.DEFAULT); - CommunityRest communityRest = communityConverter.convert(parentCommunity, DefaultProjection.DEFAULT); - CollectionRest collectionRest = collectionConverter.convert(collection, DefaultProjection.DEFAULT); // tokens String tokenAdmin = getAuthToken(admin.getEmail(), password); - String tokenEperson = getAuthToken(eperson.getEmail(), password); - // define authorizations of Admin that we know must exists Authorization authAdminSite = new Authorization(admin, administratorFeature, siteRest); - Authorization authAdminCommunity = new Authorization(admin, administratorFeature, communityRest); - Authorization authAdminCollection = new Authorization(admin, administratorFeature, collectionRest); // define authorizations of EPerson that we know not exists + Authorization authAdminComASite = new Authorization(adminComA, administratorFeature, siteRest); + Authorization authAdminColASite = new Authorization(adminColA, administratorFeature, siteRest); + Authorization authAdminItemASite = new Authorization(adminItemA, administratorFeature, siteRest); Authorization authEPersonSite = new Authorization(eperson, administratorFeature, siteRest); - Authorization authEpersonCommunity = new Authorization(eperson, administratorFeature, communityRest); - Authorization authEpersonCollection = new Authorization(eperson, administratorFeature, collectionRest); - - // define authorizations of Anonymous that we know not exists Authorization authAnonymousSite = new Authorization(null, administratorFeature, siteRest); - Authorization authAnonymousCommunity = new Authorization(null, administratorFeature, communityRest); - Authorization authAnonymousCollection = new Authorization(null, administratorFeature, collectionRest); getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authAdminSite.getID())) .andExpect(status().isOk()) .andExpect(jsonPath("$", Matchers.is(AuthorizationMatcher.matchAuthorization(authAdminSite)))); - getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authAdminCommunity.getID())) - .andExpect(status().isOk()) - .andExpect(jsonPath("$", Matchers.is(AuthorizationMatcher.matchAuthorization(authAdminCommunity)))); - - getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authAdminCollection.getID())) - .andExpect(status().isOk()) - .andExpect(jsonPath("$", Matchers.is(AuthorizationMatcher.matchAuthorization(authAdminCollection)))); - - getClient(tokenEperson).perform(get("/api/authz/authorizations/" + authEPersonSite.getID())) + getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authEPersonSite.getID())) .andExpect(status().isNotFound()); - getClient(tokenEperson).perform(get("/api/authz/authorizations/" + authEpersonCommunity.getID())) + getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authAdminComASite.getID())) .andExpect(status().isNotFound()); - getClient(tokenEperson).perform(get("/api/authz/authorizations/" + authEpersonCollection.getID())) + getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authAdminColASite.getID())) .andExpect(status().isNotFound()); - getClient().perform(get("/api/authz/authorizations/" + authAnonymousSite.getID())) + getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authAdminItemASite.getID())) .andExpect(status().isNotFound()); - getClient().perform(get("/api/authz/authorizations/" + authAnonymousCommunity.getID())) + getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authEPersonSite.getID())) .andExpect(status().isNotFound()); - getClient().perform(get("/api/authz/authorizations/" + authAnonymousCollection.getID())) + getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authAnonymousSite.getID())) .andExpect(status().isNotFound()); } + + @Test + public void itemWithAdministratorFeatureTest() throws Exception { + + ItemRest itemRestA = itemConverter.convert(itemInCollectionA, DefaultProjection.DEFAULT); + ItemRest itemRestB = itemConverter.convert(itemInCollectionB, DefaultProjection.DEFAULT); + + String tokenAdminItemA = getAuthToken(adminItemA.getEmail(), password); + String tokenAdminItemB = getAuthToken(adminItemB.getEmail(), password); + String tokenAdminColA = getAuthToken(adminColA.getEmail(), password); + String tokenAdminColB = getAuthToken(adminColB.getEmail(), password); + String tokenAdminComA = getAuthToken(adminComA.getEmail(), password); + String tokenAdminComB = getAuthToken(adminComB.getEmail(), password); + String tokenAdmin = getAuthToken(admin.getEmail(), password); + + // define authorizations that we know must exists + + Authorization authAdminSiteItemA = new Authorization(admin, administratorFeature, itemRestA); + Authorization authAdminComAItemA = new Authorization(adminComA, administratorFeature, itemRestA); + Authorization authAdminColAItemA = new Authorization(adminColA, administratorFeature, itemRestA); + Authorization authAdminItemAItemA = new Authorization(adminItemA, administratorFeature, itemRestA); + + Authorization authAdminSiteItemB = new Authorization(admin, administratorFeature, itemRestB); + Authorization authAdminComBItemB = new Authorization(adminComB, administratorFeature, itemRestB); + Authorization authAdminColBItemB = new Authorization(adminColB, administratorFeature, itemRestB); + Authorization authAdminItemBItemB = new Authorization(adminItemB, administratorFeature, itemRestB); + + + // define authorization that we know not exists + Authorization authAdminComBItemA = new Authorization(adminComB, administratorFeature, itemRestA); + Authorization authAdminColBItemA = new Authorization(adminColB, administratorFeature, itemRestA); + Authorization authAdminItemBItemA = new Authorization(adminItemB, administratorFeature, itemRestA); + Authorization authEPersonItemA = new Authorization(eperson, administratorFeature, itemRestA); + Authorization authAnonymousItemA = new Authorization(null, administratorFeature, itemRestA); + + + + getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authAdminSiteItemA.getID())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", Matchers.is( + AuthorizationMatcher.matchAuthorization(authAdminSiteItemA)))); + + getClient(tokenAdminComA).perform(get("/api/authz/authorizations/" + authAdminComAItemA.getID())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", Matchers.is( + AuthorizationMatcher.matchAuthorization(authAdminComAItemA)))); + + getClient(tokenAdminColA).perform(get("/api/authz/authorizations/" + authAdminColAItemA.getID())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", Matchers.is( + AuthorizationMatcher.matchAuthorization(authAdminColAItemA)))); + + getClient(tokenAdminItemA).perform(get("/api/authz/authorizations/" + authAdminItemAItemA.getID())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", Matchers.is( + AuthorizationMatcher.matchAuthorization(authAdminItemAItemA)))); + + getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authAdminSiteItemB.getID())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", Matchers.is( + AuthorizationMatcher.matchAuthorization(authAdminSiteItemB)))); + + getClient(tokenAdminComB).perform(get("/api/authz/authorizations/" + authAdminComBItemB.getID())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", Matchers.is( + AuthorizationMatcher.matchAuthorization(authAdminComBItemB)))); + + getClient(tokenAdminColB).perform(get("/api/authz/authorizations/" + authAdminColBItemB.getID())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", Matchers.is( + AuthorizationMatcher.matchAuthorization(authAdminColBItemB)))); + + getClient(tokenAdminItemB).perform(get("/api/authz/authorizations/" + authAdminItemBItemB.getID())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", Matchers.is( + AuthorizationMatcher.matchAuthorization(authAdminItemBItemB)))); + + + + getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authAdminComBItemA.getID())) + .andExpect(status().isNotFound()); + getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authAdminColBItemA.getID())) + .andExpect(status().isNotFound()); + getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authAdminItemBItemA.getID())) + .andExpect(status().isNotFound()); + getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authEPersonItemA.getID())) + .andExpect(status().isNotFound()); + getClient(tokenAdmin).perform(get("/api/authz/authorizations/" + authAnonymousItemA.getID())) + .andExpect(status().isNotFound()); + } } From 240b327bb3899e1aff84dc3e153640a9ea03f169 Mon Sep 17 00:00:00 2001 From: Raf Ponsaerts Date: Tue, 11 Aug 2020 09:42:07 +0200 Subject: [PATCH 59/96] [Task 72232] altered email param usage for curation scripts when being called from cli or rest --- .../main/java/org/dspace/curate/Curation.java | 364 ++++++++++++++++++ .../java/org/dspace/curate/CurationCli.java | 333 +--------------- .../CurationCliScriptConfiguration.java | 22 ++ .../dspace/curate/CurationClientOptions.java | 1 - .../curate/CurationScriptConfiguration.java | 4 +- .../config/spring/api/scripts.xml | 2 +- .../java/org/dspace/curate/CurationTest.java | 78 ++++ .../impl/RestDSpaceRunnableHandler.java | 7 +- .../config/spring/rest/scripts.xml | 20 + .../org/dspace/curate/CurationScriptIT.java | 152 +++----- dspace/config/spring/api/scripts.xml | 2 +- dspace/config/spring/rest/scripts.xml | 5 + 12 files changed, 556 insertions(+), 434 deletions(-) create mode 100644 dspace-api/src/main/java/org/dspace/curate/Curation.java create mode 100644 dspace-api/src/main/java/org/dspace/curate/CurationCliScriptConfiguration.java create mode 100644 dspace-api/src/test/java/org/dspace/curate/CurationTest.java create mode 100644 dspace-server-webapp/src/test/data/dspaceFolder/config/spring/rest/scripts.xml diff --git a/dspace-api/src/main/java/org/dspace/curate/Curation.java b/dspace-api/src/main/java/org/dspace/curate/Curation.java new file mode 100644 index 0000000000..50ad2d1197 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/curate/Curation.java @@ -0,0 +1,364 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.curate; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.PrintStream; +import java.io.Writer; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.UUID; + +import org.apache.commons.cli.ParseException; +import org.apache.commons.io.output.NullOutputStream; +import org.dspace.authorize.AuthorizeException; +import org.dspace.content.DSpaceObject; +import org.dspace.content.factory.ContentServiceFactory; +import org.dspace.core.Context; +import org.dspace.core.factory.CoreServiceFactory; +import org.dspace.curate.factory.CurateServiceFactory; +import org.dspace.eperson.EPerson; +import org.dspace.eperson.factory.EPersonServiceFactory; +import org.dspace.eperson.service.EPersonService; +import org.dspace.handle.factory.HandleServiceFactory; +import org.dspace.handle.service.HandleService; +import org.dspace.scripts.DSpaceRunnable; +import org.dspace.utils.DSpace; + +/** + * CurationCli provides command-line access to Curation tools and processes. + * + * @author richardrodgers + */ +public class Curation extends DSpaceRunnable { + + protected EPersonService ePersonService = EPersonServiceFactory.getInstance().getEPersonService(); + + protected Context context; + private CurationClientOptions curationClientOptions; + + private String task; + private String taskFile; + private String id; + private String queue; + private String scope; + private String reporter; + private Map parameters; + private boolean verbose; + + @Override + public void internalRun() throws Exception { + if (curationClientOptions == CurationClientOptions.HELP) { + printHelp(); + return; + } + + Curator curator = initCurator(); + + // load curation tasks + if (curationClientOptions == CurationClientOptions.TASK) { + long start = System.currentTimeMillis(); + handleCurationTask(curator); + this.endScript(start); + } + + // process task queue + if (curationClientOptions == CurationClientOptions.QUEUE) { + // process the task queue + TaskQueue taskQueue = (TaskQueue) CoreServiceFactory.getInstance().getPluginService() + .getSinglePlugin(TaskQueue.class); + if (taskQueue == null) { + super.handler.logError("No implementation configured for queue"); + throw new UnsupportedOperationException("No queue service available"); + } + long timeRun = this.runQueue(taskQueue, curator); + this.endScript(timeRun); + } + } + + /** + * Does the curation task (-t) or the task in the given file (-T). + * Checks: + * - if required option -i is missing. + * - if option -t has a valid task option + */ + private void handleCurationTask(Curator curator) throws IOException, SQLException { + String taskName; + if (commandLine.hasOption('t')) { + if (verbose) { + handler.logInfo("Adding task: " + this.task); + } + curator.addTask(this.task); + if (verbose && !curator.hasTask(this.task)) { + handler.logInfo("Task: " + this.task + " not resolved"); + } + } else if (commandLine.hasOption('T')) { + // load taskFile + BufferedReader reader = null; + try { + reader = new BufferedReader(new FileReader(this.taskFile)); + while ((taskName = reader.readLine()) != null) { + if (verbose) { + super.handler.logInfo("Adding task: " + taskName); + } + curator.addTask(taskName); + } + } finally { + if (reader != null) { + reader.close(); + } + } + } + // run tasks against object + if (verbose) { + super.handler.logInfo("Starting curation"); + super.handler.logInfo("Curating id: " + this.id); + } + if ("all".equals(this.id)) { + // run on whole Site + curator.curate(context, + ContentServiceFactory.getInstance().getSiteService().findSite(context).getHandle()); + } else { + curator.curate(context, this.id); + } + } + + /** + * Runs task queue (-q set) + * + * @param queue The task queue + * @param curator The curator + * @return Time when queue started + */ + private long runQueue(TaskQueue queue, Curator curator) throws SQLException, AuthorizeException, IOException { + // use current time as our reader 'ticket' + long ticket = System.currentTimeMillis(); + Iterator entryIter = queue.dequeue(this.queue, ticket).iterator(); + while (entryIter.hasNext()) { + TaskQueueEntry entry = entryIter.next(); + if (verbose) { + super.handler.logInfo("Curating id: " + entry.getObjectId()); + } + curator.clear(); + // does entry relate to a DSO or workflow object? + if (entry.getObjectId().indexOf('/') > 0) { + for (String taskName : entry.getTaskNames()) { + curator.addTask(taskName); + } + curator.curate(context, entry.getObjectId()); + } else { + // make eperson who queued task the effective user + EPerson agent = ePersonService.findByEmail(context, entry.getEpersonId()); + if (agent != null) { + context.setCurrentUser(agent); + } + CurateServiceFactory.getInstance().getWorkflowCuratorService() + .curate(curator, context, entry.getObjectId()); + } + } + queue.release(this.queue, ticket, true); + return ticket; + } + + /** + * End of curation script; logs script time if -v verbose is set + * + * @param timeRun Time script was started + * @throws SQLException If DSpace contextx can't complete + */ + private void endScript(long timeRun) throws SQLException { + context.complete(); + if (verbose) { + long elapsed = System.currentTimeMillis() - timeRun; + this.handler.logInfo("Ending curation. Elapsed time: " + elapsed); + } + } + + /** + * Initialize the curator with command line variables + * + * @return Initialised curator + * @throws FileNotFoundException If file of command line variable -r reporter is not found + */ + private Curator initCurator() throws FileNotFoundException { + Curator curator = new Curator(); + OutputStream reporterStream; + if (null == this.reporter) { + reporterStream = new NullOutputStream(); + } else if ("-".equals(this.reporter)) { + reporterStream = System.out; + } else { + reporterStream = new PrintStream(this.reporter); + } + Writer reportWriter = new OutputStreamWriter(reporterStream); + curator.setReporter(reportWriter); + + if (this.scope != null) { + Curator.TxScope txScope = Curator.TxScope.valueOf(this.scope.toUpperCase()); + curator.setTransactionScope(txScope); + } + + curator.addParameters(parameters); + // we are operating in batch mode, if anyone cares. + curator.setInvoked(Curator.Invoked.BATCH); + return curator; + } + + @Override + public void printHelp() { + super.printHelp(); + super.handler.logInfo("\nwhole repo: CurationCli -t estimate -i all"); + super.handler.logInfo("single item: CurationCli -t generate -i itemId"); + super.handler.logInfo("task queue: CurationCli -q monthly"); + } + + @Override + public CurationScriptConfiguration getScriptConfiguration() { + return new DSpace().getServiceManager().getServiceByName("curate", CurationScriptConfiguration.class); + } + + @Override + public void setup() throws ParseException { + assignCurrentUserInContext(); + this.curationClientOptions = CurationClientOptions.getClientOption(commandLine); + + if (this.curationClientOptions != null) { + this.initGeneralLineOptionsAndCheckIfValid(); + if (curationClientOptions == CurationClientOptions.TASK) { + this.initTaskLineOptionsAndCheckIfValid(); + } else if (curationClientOptions == CurationClientOptions.QUEUE) { + this.queue = this.commandLine.getOptionValue('q'); + } + } else { + throw new IllegalArgumentException("[--help || --task|--taskfile <> -identifier <> || -queue <> ] must be" + + " specified"); + } + } + + protected void assignCurrentUserInContext() throws ParseException { + UUID currentUserUuid = this.getEpersonIdentifier(); + try { + this.context = new Context(Context.Mode.BATCH_EDIT); + EPerson eperson = ePersonService.find(context, currentUserUuid); + if (eperson == null) { + super.handler.logError("EPerson not found: " + currentUserUuid); + throw new IllegalArgumentException("Unable to find a user with uuid: " + currentUserUuid); + } + this.context.setCurrentUser(eperson); + } catch (SQLException e) { + handler.handleException("Something went wrong trying to fetch eperson for uuid: " + currentUserUuid, e); + } + } + + /** + * Fills in some optional command line options. + * Checks if there are missing required options or invalid values for options. + */ + private void initGeneralLineOptionsAndCheckIfValid() { + // report file + if (this.commandLine.hasOption('r')) { + this.reporter = this.commandLine.getOptionValue('r'); + } + + // parameters + this.parameters = new HashMap<>(); + if (this.commandLine.hasOption('p')) { + for (String parameter : this.commandLine.getOptionValues('p')) { + String[] parts = parameter.split("=", 2); + String name = parts[0].trim(); + String value; + if (parts.length > 1) { + value = parts[1].trim(); + } else { + value = "true"; + } + this.parameters.put(name, value); + } + } + + // verbose + verbose = false; + if (commandLine.hasOption('v')) { + verbose = true; + } + + // scope + if (this.commandLine.getOptionValue('s') != null) { + this.scope = this.commandLine.getOptionValue('s'); + if (this.scope != null && Curator.TxScope.valueOf(this.scope.toUpperCase()) == null) { + this.handler.logError("Bad transaction scope '" + this.scope + "': only 'object', 'curation' or " + + "'open' recognized"); + throw new IllegalArgumentException( + "Bad transaction scope '" + this.scope + "': only 'object', 'curation' or " + + "'open' recognized"); + } + } + } + + /** + * Fills in required command line options for the task or taskFile option. + * Checks if there are is a missing required -i option and if -i is either 'all' or a valid dso handle. + * Checks if -t task has a valid task option. + * Checks if -T taskfile is a valid file. + */ + private void initTaskLineOptionsAndCheckIfValid() { + // task or taskFile + if (this.commandLine.hasOption('t')) { + this.task = this.commandLine.getOptionValue('t'); + if (!CurationClientOptions.getTaskOptions().contains(this.task)) { + super.handler + .logError("-t task must be one of: " + CurationClientOptions.getTaskOptions()); + throw new IllegalArgumentException( + "-t task must be one of: " + CurationClientOptions.getTaskOptions()); + } + } else if (this.commandLine.hasOption('T')) { + this.taskFile = this.commandLine.getOptionValue('T'); + if (!(new File(this.taskFile).isFile())) { + super.handler + .logError("-T taskFile must be valid file: " + this.taskFile); + throw new IllegalArgumentException("-T taskFile must be valid file: " + this.taskFile); + } + } + + if (this.commandLine.hasOption('i')) { + this.id = this.commandLine.getOptionValue('i').toLowerCase(); + if (!this.id.equalsIgnoreCase("all")) { + HandleService handleService = HandleServiceFactory.getInstance().getHandleService(); + DSpaceObject dso; + try { + dso = handleService.resolveToObject(this.context, id); + } catch (SQLException e) { + super.handler.logError("SQLException trying to resolve handle " + id + " to a valid dso"); + throw new IllegalArgumentException( + "SQLException trying to resolve handle " + id + " to a valid dso"); + } + if (dso == null) { + super.handler.logError("Id must be specified: a valid dso handle or 'all'; " + this.id + " could " + + "not be resolved to valid dso handle"); + throw new IllegalArgumentException( + "Id must be specified: a valid dso handle or 'all'; " + this.id + " could " + + "not be resolved to valid dso handle"); + } + } + } else { + super.handler.logError("Id must be specified: a handle, 'all', or no -i and a -q task queue (-h for " + + "help)"); + throw new IllegalArgumentException( + "Id must be specified: a handle, 'all', or no -i and a -q task queue (-h for " + + "help)"); + } + } +} diff --git a/dspace-api/src/main/java/org/dspace/curate/CurationCli.java b/dspace-api/src/main/java/org/dspace/curate/CurationCli.java index 8f5d91cc1c..cd197b5a16 100644 --- a/dspace-api/src/main/java/org/dspace/curate/CurationCli.java +++ b/dspace-api/src/main/java/org/dspace/curate/CurationCli.java @@ -7,229 +7,16 @@ */ package org.dspace.curate; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.IOException; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.PrintStream; -import java.io.Writer; import java.sql.SQLException; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import org.apache.commons.io.output.NullOutputStream; -import org.dspace.authorize.AuthorizeException; -import org.dspace.content.DSpaceObject; -import org.dspace.content.factory.ContentServiceFactory; +import org.apache.commons.cli.ParseException; import org.dspace.core.Context; -import org.dspace.core.factory.CoreServiceFactory; -import org.dspace.curate.factory.CurateServiceFactory; import org.dspace.eperson.EPerson; -import org.dspace.eperson.factory.EPersonServiceFactory; -import org.dspace.eperson.service.EPersonService; -import org.dspace.handle.factory.HandleServiceFactory; -import org.dspace.handle.service.HandleService; -import org.dspace.scripts.DSpaceRunnable; -import org.dspace.utils.DSpace; -/** - * CurationCli provides command-line access to Curation tools and processes. - * - * @author richardrodgers - */ -public class CurationCli extends DSpaceRunnable { - - private EPersonService ePersonService = EPersonServiceFactory.getInstance().getEPersonService(); - - private Context context; - private CurationClientOptions curationClientOptions; - - private String task; - private String taskFile; - private String id; - private String queue; - private String scope; - private String reporter; - private Map parameters; - private boolean verbose; +public class CurationCli extends Curation { @Override - public void internalRun() throws Exception { - if (curationClientOptions == CurationClientOptions.HELP) { - printHelp(); - return; - } - - Curator curator = initCurator(); - - // load curation tasks - if (curationClientOptions == CurationClientOptions.TASK) { - long start = System.currentTimeMillis(); - handleCurationTask(curator); - this.endScript(start); - } - - // process task queue - if (curationClientOptions == CurationClientOptions.QUEUE) { - // process the task queue - TaskQueue taskQueue = (TaskQueue) CoreServiceFactory.getInstance().getPluginService() - .getSinglePlugin(TaskQueue.class); - if (taskQueue == null) { - super.handler.logError("No implementation configured for queue"); - throw new UnsupportedOperationException("No queue service available"); - } - long timeRun = this.runQueue(taskQueue, curator); - this.endScript(timeRun); - } - } - - /** - * Does the curation task (-t) or the task in the given file (-T). - * Checks: - * - if required option -i is missing. - * - if option -t has a valid task option - */ - private void handleCurationTask(Curator curator) throws IOException, SQLException { - String taskName; - if (commandLine.hasOption('t')) { - if (verbose) { - handler.logInfo("Adding task: " + this.task); - } - curator.addTask(this.task); - if (verbose && !curator.hasTask(this.task)) { - handler.logInfo("Task: " + this.task + " not resolved"); - } - } else if (commandLine.hasOption('T')) { - // load taskFile - BufferedReader reader = null; - try { - reader = new BufferedReader(new FileReader(this.taskFile)); - while ((taskName = reader.readLine()) != null) { - if (verbose) { - super.handler.logInfo("Adding task: " + taskName); - } - curator.addTask(taskName); - } - } finally { - if (reader != null) { - reader.close(); - } - } - } - // run tasks against object - if (verbose) { - super.handler.logInfo("Starting curation"); - super.handler.logInfo("Curating id: " + this.id); - } - if ("all".equals(this.id)) { - // run on whole Site - curator.curate(context, - ContentServiceFactory.getInstance().getSiteService().findSite(context).getHandle()); - } else { - curator.curate(context, this.id); - } - } - - /** - * Runs task queue (-q set) - * - * @param queue The task queue - * @param curator The curator - * @return Time when queue started - */ - private long runQueue(TaskQueue queue, Curator curator) throws SQLException, AuthorizeException, IOException { - // use current time as our reader 'ticket' - long ticket = System.currentTimeMillis(); - Iterator entryIter = queue.dequeue(this.queue, ticket).iterator(); - while (entryIter.hasNext()) { - TaskQueueEntry entry = entryIter.next(); - if (verbose) { - super.handler.logInfo("Curating id: " + entry.getObjectId()); - } - curator.clear(); - // does entry relate to a DSO or workflow object? - if (entry.getObjectId().indexOf('/') > 0) { - for (String taskName : entry.getTaskNames()) { - curator.addTask(taskName); - } - curator.curate(context, entry.getObjectId()); - } else { - // make eperson who queued task the effective user - EPerson agent = ePersonService.findByEmail(context, entry.getEpersonId()); - if (agent != null) { - context.setCurrentUser(agent); - } - CurateServiceFactory.getInstance().getWorkflowCuratorService() - .curate(curator, context, entry.getObjectId()); - } - } - queue.release(this.queue, ticket, true); - return ticket; - } - - /** - * End of curation script; logs script time if -v verbose is set - * - * @param timeRun Time script was started - * @throws SQLException If DSpace contextx can't complete - */ - private void endScript(long timeRun) throws SQLException { - context.complete(); - if (verbose) { - long elapsed = System.currentTimeMillis() - timeRun; - this.handler.logInfo("Ending curation. Elapsed time: " + elapsed); - } - } - - /** - * Initialize the curator with command line variables - * - * @return Initialised curator - * @throws FileNotFoundException If file of command line variable -r reporter is not found - */ - private Curator initCurator() throws FileNotFoundException { - Curator curator = new Curator(); - OutputStream reporterStream; - if (null == this.reporter) { - reporterStream = new NullOutputStream(); - } else if ("-".equals(this.reporter)) { - reporterStream = System.out; - } else { - reporterStream = new PrintStream(this.reporter); - } - Writer reportWriter = new OutputStreamWriter(reporterStream); - curator.setReporter(reportWriter); - - if (this.scope != null) { - Curator.TxScope txScope = Curator.TxScope.valueOf(this.scope.toUpperCase()); - curator.setTransactionScope(txScope); - } - - curator.addParameters(parameters); - // we are operating in batch mode, if anyone cares. - curator.setInvoked(Curator.Invoked.BATCH); - return curator; - } - - @Override - public void printHelp() { - super.printHelp(); - super.handler.logInfo("\nwhole repo: CurationCli -t estimate -i all"); - super.handler.logInfo("single item: CurationCli -t generate -i itemId"); - super.handler.logInfo("task queue: CurationCli -q monthly"); - } - - @Override - public CurationScriptConfiguration getScriptConfiguration() { - return new DSpace().getServiceManager().getServiceByName("curate", CurationScriptConfiguration.class); - } - - @Override - public void setup() { + protected void assignCurrentUserInContext() throws ParseException { if (this.commandLine.hasOption('e')) { String ePersonEmail = this.commandLine.getOptionValue('e'); this.context = new Context(Context.Mode.BATCH_EDIT); @@ -244,119 +31,7 @@ public class CurationCli extends DSpaceRunnable { throw new IllegalArgumentException("SQLException trying to find user with email: " + ePersonEmail); } } else { - throw new IllegalArgumentException("Needs an -e to set eperson (admin)"); - } - this.curationClientOptions = CurationClientOptions.getClientOption(commandLine); - - if (this.curationClientOptions != null) { - this.initGeneralLineOptionsAndCheckIfValid(); - if (curationClientOptions == CurationClientOptions.TASK) { - this.initTaskLineOptionsAndCheckIfValid(); - } else if (curationClientOptions == CurationClientOptions.QUEUE) { - this.queue = this.commandLine.getOptionValue('q'); - } - } else { - throw new IllegalArgumentException("[--help || --task|--taskfile <> -identifier <> || -queue <> ] must be" + - " specified"); - } - } - - /** - * Fills in some optional command line options. - * Checks if there are missing required options or invalid values for options. - */ - private void initGeneralLineOptionsAndCheckIfValid() { - // report file - if (this.commandLine.hasOption('r')) { - this.reporter = this.commandLine.getOptionValue('r'); - } - - // parameters - this.parameters = new HashMap<>(); - if (this.commandLine.hasOption('p')) { - for (String parameter : this.commandLine.getOptionValues('p')) { - String[] parts = parameter.split("=", 2); - String name = parts[0].trim(); - String value; - if (parts.length > 1) { - value = parts[1].trim(); - } else { - value = "true"; - } - this.parameters.put(name, value); - } - } - - // verbose - verbose = false; - if (commandLine.hasOption('v')) { - verbose = true; - } - - // scope - if (this.commandLine.getOptionValue('s') != null) { - this.scope = this.commandLine.getOptionValue('s'); - if (this.scope != null && Curator.TxScope.valueOf(this.scope.toUpperCase()) == null) { - this.handler.logError("Bad transaction scope '" + this.scope + "': only 'object', 'curation' or " + - "'open' recognized"); - throw new IllegalArgumentException( - "Bad transaction scope '" + this.scope + "': only 'object', 'curation' or " + - "'open' recognized"); - } - } - } - - /** - * Fills in required command line options for the task or taskFile option. - * Checks if there are is a missing required -i option and if -i is either 'all' or a valid dso handle. - * Checks if -t task has a valid task option. - * Checks if -T taskfile is a valid file. - */ - private void initTaskLineOptionsAndCheckIfValid() { - // task or taskFile - if (this.commandLine.hasOption('t')) { - this.task = this.commandLine.getOptionValue('t'); - if (!CurationClientOptions.getTaskOptions().contains(this.task)) { - super.handler - .logError("-t task must be one of: " + CurationClientOptions.getTaskOptions()); - throw new IllegalArgumentException( - "-t task must be one of: " + CurationClientOptions.getTaskOptions()); - } - } else if (this.commandLine.hasOption('T')) { - this.taskFile = this.commandLine.getOptionValue('T'); - if (!(new File(this.taskFile).isFile())) { - super.handler - .logError("-T taskFile must be valid file: " + this.taskFile); - throw new IllegalArgumentException("-T taskFile must be valid file: " + this.taskFile); - } - } - - if (this.commandLine.hasOption('i')) { - this.id = this.commandLine.getOptionValue('i').toLowerCase(); - if (!this.id.equalsIgnoreCase("all")) { - HandleService handleService = HandleServiceFactory.getInstance().getHandleService(); - DSpaceObject dso; - try { - dso = handleService.resolveToObject(this.context, id); - } catch (SQLException e) { - super.handler.logError("SQLException trying to resolve handle " + id + " to a valid dso"); - throw new IllegalArgumentException( - "SQLException trying to resolve handle " + id + " to a valid dso"); - } - if (dso == null) { - super.handler.logError("Id must be specified: a valid dso handle or 'all'; " + this.id + " could " + - "not be resolved to valid dso handle"); - throw new IllegalArgumentException( - "Id must be specified: a valid dso handle or 'all'; " + this.id + " could " + - "not be resolved to valid dso handle"); - } - } - } else { - super.handler.logError("Id must be specified: a handle, 'all', or no -i and a -q task queue (-h for " + - "help)"); - throw new IllegalArgumentException( - "Id must be specified: a handle, 'all', or no -i and a -q task queue (-h for " + - "help)"); + throw new ParseException("Required parameter -e missing!"); } } } diff --git a/dspace-api/src/main/java/org/dspace/curate/CurationCliScriptConfiguration.java b/dspace-api/src/main/java/org/dspace/curate/CurationCliScriptConfiguration.java new file mode 100644 index 0000000000..5e1b159bba --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/curate/CurationCliScriptConfiguration.java @@ -0,0 +1,22 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.curate; + +import org.apache.commons.cli.Options; + +public class CurationCliScriptConfiguration extends CurationScriptConfiguration { + + @Override + public Options getOptions() { + options = super.getOptions(); + options.addOption("e", "eperson", true, "email address of curating eperson"); + options.getOption("e").setType(String.class); + options.getOption("e").setRequired(true); + return options; + } +} diff --git a/dspace-api/src/main/java/org/dspace/curate/CurationClientOptions.java b/dspace-api/src/main/java/org/dspace/curate/CurationClientOptions.java index 7daf107aad..1db1ac600a 100644 --- a/dspace-api/src/main/java/org/dspace/curate/CurationClientOptions.java +++ b/dspace-api/src/main/java/org/dspace/curate/CurationClientOptions.java @@ -54,7 +54,6 @@ public enum CurationClientOptions { "Id (handle) of object to perform task on, or 'all' to perform on whole repository"); options.addOption("p", "parameter", true, "a task parameter 'NAME=VALUE'"); options.addOption("q", "queue", true, "name of task queue to process"); - options.addOption("e", "eperson", true, "email address of curating eperson"); options.addOption("r", "reporter", true, "relative or absolute path to the desired report file. Use '-' to report to console. If absent, no " + "reporting"); diff --git a/dspace-api/src/main/java/org/dspace/curate/CurationScriptConfiguration.java b/dspace-api/src/main/java/org/dspace/curate/CurationScriptConfiguration.java index 785926908e..fefb4eb768 100644 --- a/dspace-api/src/main/java/org/dspace/curate/CurationScriptConfiguration.java +++ b/dspace-api/src/main/java/org/dspace/curate/CurationScriptConfiguration.java @@ -16,11 +16,11 @@ import org.dspace.scripts.configuration.ScriptConfiguration; import org.springframework.beans.factory.annotation.Autowired; /** - * The {@link ScriptConfiguration} for the {@link CurationCli} script + * The {@link ScriptConfiguration} for the {@link Curation} script * * @author Maria Verdonck (Atmire) on 23/06/2020 */ -public class CurationScriptConfiguration extends ScriptConfiguration { +public class CurationScriptConfiguration extends ScriptConfiguration { @Autowired private AuthorizeService authorizeService; diff --git a/dspace-api/src/test/data/dspaceFolder/config/spring/api/scripts.xml b/dspace-api/src/test/data/dspaceFolder/config/spring/api/scripts.xml index 76cb57a40d..f9043dca04 100644 --- a/dspace-api/src/test/data/dspaceFolder/config/spring/api/scripts.xml +++ b/dspace-api/src/test/data/dspaceFolder/config/spring/api/scripts.xml @@ -19,7 +19,7 @@ - + diff --git a/dspace-api/src/test/java/org/dspace/curate/CurationTest.java b/dspace-api/src/test/java/org/dspace/curate/CurationTest.java new file mode 100644 index 0000000000..184bde0b0c --- /dev/null +++ b/dspace-api/src/test/java/org/dspace/curate/CurationTest.java @@ -0,0 +1,78 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.curate; + +import org.apache.commons.cli.ParseException; +import org.dspace.AbstractIntegrationTestWithDatabase; +import org.dspace.app.scripts.handler.impl.TestDSpaceRunnableHandler; +import org.dspace.builder.CollectionBuilder; +import org.dspace.builder.CommunityBuilder; +import org.dspace.content.Collection; +import org.dspace.content.Community; +import org.dspace.scripts.DSpaceRunnable; +import org.dspace.scripts.configuration.ScriptConfiguration; +import org.dspace.scripts.factory.ScriptServiceFactory; +import org.dspace.scripts.service.ScriptService; +import org.junit.Ignore; +import org.junit.Test; + +public class CurationTest extends AbstractIntegrationTestWithDatabase { + + @Test(expected = ParseException.class) + public void curationWithoutEPersonParameterTest() throws Exception { + + context.turnOffAuthorisationSystem(); + Community community = CommunityBuilder.createCommunity(context) + .build(); + Collection collection = CollectionBuilder.createCollection(context, community) + .build(); + context.restoreAuthSystemState(); + String[] args = new String[] {"curate", "-t", CurationClientOptions.getTaskOptions().get(0), + "-i", collection.getHandle()}; + TestDSpaceRunnableHandler testDSpaceRunnableHandler = new TestDSpaceRunnableHandler(); + + ScriptService scriptService = ScriptServiceFactory.getInstance().getScriptService(); + ScriptConfiguration scriptConfiguration = scriptService.getScriptConfiguration(args[0]); + + DSpaceRunnable script = null; + if (scriptConfiguration != null) { + script = scriptService.createDSpaceRunnableForScriptConfiguration(scriptConfiguration); + } + if (script != null) { + script.initialize(args, testDSpaceRunnableHandler, null); + script.run(); + } + } + + @Test + @Ignore + public void curationWithEPersonParameterTest() throws Exception { + + context.turnOffAuthorisationSystem(); + Community community = CommunityBuilder.createCommunity(context) + .build(); + Collection collection = CollectionBuilder.createCollection(context, community) + .build(); + context.restoreAuthSystemState(); + String[] args = new String[] {"curate", "-e", "admin@email.com", "-t", + CurationClientOptions.getTaskOptions().get(0), "-i", collection.getHandle()}; + TestDSpaceRunnableHandler testDSpaceRunnableHandler = new TestDSpaceRunnableHandler(); + + ScriptService scriptService = ScriptServiceFactory.getInstance().getScriptService(); + ScriptConfiguration scriptConfiguration = scriptService.getScriptConfiguration(args[0]); + + DSpaceRunnable script = null; + if (scriptConfiguration != null) { + script = scriptService.createDSpaceRunnableForScriptConfiguration(scriptConfiguration); + } + if (script != null) { + script.initialize(args, testDSpaceRunnableHandler, null); + script.run(); + } + } +} diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java index f2080dcd84..d61b87c7d4 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java @@ -33,7 +33,7 @@ import org.dspace.scripts.factory.ScriptServiceFactory; import org.dspace.scripts.handler.DSpaceRunnableHandler; import org.dspace.scripts.service.ProcessService; import org.dspace.utils.DSpace; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.core.task.TaskExecutor; /** * The {@link DSpaceRunnableHandler} dealing with Scripts started from the REST api @@ -231,9 +231,8 @@ public class RestDSpaceRunnableHandler implements DSpaceRunnableHandler { * @param script The script to be ran */ public void schedule(DSpaceRunnable script) { - ThreadPoolTaskExecutor taskExecutor = new DSpace().getServiceManager() - .getServiceByName("dspaceRunnableThreadExecutor", - ThreadPoolTaskExecutor.class); + TaskExecutor taskExecutor = new DSpace().getServiceManager() + .getServiceByName("dspaceRunnableThreadExecutor", TaskExecutor.class); Context context = new Context(); try { Process process = processService.find(context, processId); diff --git a/dspace-server-webapp/src/test/data/dspaceFolder/config/spring/rest/scripts.xml b/dspace-server-webapp/src/test/data/dspaceFolder/config/spring/rest/scripts.xml new file mode 100644 index 0000000000..5ddbe7e24d --- /dev/null +++ b/dspace-server-webapp/src/test/data/dspaceFolder/config/spring/rest/scripts.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dspace-server-webapp/src/test/java/org/dspace/curate/CurationScriptIT.java b/dspace-server-webapp/src/test/java/org/dspace/curate/CurationScriptIT.java index 58a92f7505..66c0319857 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/curate/CurationScriptIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/curate/CurationScriptIT.java @@ -40,7 +40,7 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; /** - * IT for {@link CurationCli} + * IT for {@link Curation} * * @author Maria Verdonck (Atmire) on 24/06/2020 */ @@ -75,7 +75,6 @@ public class CurationScriptIT extends AbstractControllerIntegrationTest { LinkedList parameters = new LinkedList<>(); - parameters.add(new DSpaceCommandLineParameter("-e", admin.getEmail())); parameters.add(new DSpaceCommandLineParameter("-i", publicItem1.getHandle())); parameters.add(new DSpaceCommandLineParameter("-t", "invalidTaskOption")); @@ -95,98 +94,12 @@ public class CurationScriptIT extends AbstractControllerIntegrationTest { .andExpect(status().isBadRequest()); } - @Test - public void curateScript_MissingEperson() throws Exception { - context.turnOffAuthorisationSystem(); - - String token = getAuthToken(admin.getEmail(), password); - - parentCommunity = CommunityBuilder.createCommunity(context) - .withName("Parent Community") - .build(); - Community child1 = CommunityBuilder.createSubCommunity(context, parentCommunity) - .withName("Sub Community") - .build(); - Collection col1 = CollectionBuilder.createCollection(context, child1).withName("Collection 1").build(); - - Item publicItem1 = ItemBuilder.createItem(context, col1) - .withTitle("Public item 1") - .withIssueDate("2017-10-17") - .withAuthor("Smith, Donald").withAuthor("Doe, John") - .withSubject("ExtraEntry") - .build(); - - LinkedList parameters = new LinkedList<>(); - - parameters.add(new DSpaceCommandLineParameter("-i", publicItem1.getHandle())); - parameters.add(new DSpaceCommandLineParameter("-t", CurationClientOptions.getTaskOptions().get(0))); - - List list = parameters.stream() - .map(dSpaceCommandLineParameter -> dSpaceRunnableParameterConverter - .convert(dSpaceCommandLineParameter, Projection.DEFAULT)) - .collect(Collectors.toList()); - - context.restoreAuthSystemState(); - - // Request with missing required -e - getClient(token) - .perform(post(CURATE_SCRIPT_ENDPOINT).contentType("multipart/form-data") - .param("properties", - new Gson().toJson(list))) - // Illegal Argument Exception - .andExpect(status().isBadRequest()); - } - - @Test - public void curateScript_NonExistentEPerson() throws Exception { - context.turnOffAuthorisationSystem(); - - String token = getAuthToken(admin.getEmail(), password); - - parentCommunity = CommunityBuilder.createCommunity(context) - .withName("Parent Community") - .build(); - Community child1 = CommunityBuilder.createSubCommunity(context, parentCommunity) - .withName("Sub Community") - .build(); - Collection col1 = CollectionBuilder.createCollection(context, child1).withName("Collection 1").build(); - - Item publicItem1 = ItemBuilder.createItem(context, col1) - .withTitle("Public item 1") - .withIssueDate("2017-10-17") - .withAuthor("Smith, Donald").withAuthor("Doe, John") - .withSubject("ExtraEntry") - .build(); - - LinkedList parameters = new LinkedList<>(); - - parameters.add(new DSpaceCommandLineParameter("-e", "nonExistentEmail@test.com")); - parameters.add(new DSpaceCommandLineParameter("-i", publicItem1.getHandle())); - parameters.add(new DSpaceCommandLineParameter("-t", CurationClientOptions.getTaskOptions().get(0))); - - List list = parameters.stream() - .map(dSpaceCommandLineParameter -> dSpaceRunnableParameterConverter - .convert(dSpaceCommandLineParameter, Projection.DEFAULT)) - .collect(Collectors.toList()); - - context.restoreAuthSystemState(); - - // Request with -e - getClient(token) - .perform(post(CURATE_SCRIPT_ENDPOINT).contentType("multipart/form-data") - .param("properties", - new Gson().toJson(list))) - // Illegal Argument Exception - .andExpect(status().isBadRequest()); - } - @Test public void curateScript_MissingHandle() throws Exception { String token = getAuthToken(admin.getEmail(), password); LinkedList parameters = new LinkedList<>(); - parameters.add(new DSpaceCommandLineParameter("-e", admin.getEmail())); parameters.add(new DSpaceCommandLineParameter("-t", CurationClientOptions.getTaskOptions().get(0))); List list = parameters.stream() @@ -210,7 +123,6 @@ public class CurationScriptIT extends AbstractControllerIntegrationTest { LinkedList parameters = new LinkedList<>(); parameters.add(new DSpaceCommandLineParameter("-i", "invalidhandle")); - parameters.add(new DSpaceCommandLineParameter("-e", admin.getEmail())); parameters.add(new DSpaceCommandLineParameter("-t", CurationClientOptions.getTaskOptions().get(0))); List list = parameters.stream() @@ -250,7 +162,6 @@ public class CurationScriptIT extends AbstractControllerIntegrationTest { LinkedList parameters = new LinkedList<>(); - parameters.add(new DSpaceCommandLineParameter("-e", admin.getEmail())); parameters.add(new DSpaceCommandLineParameter("-i", publicItem1.getHandle())); List list = parameters.stream() @@ -275,7 +186,6 @@ public class CurationScriptIT extends AbstractControllerIntegrationTest { LinkedList parameters = new LinkedList<>(); - parameters.add(new DSpaceCommandLineParameter("-e", admin.getEmail())); parameters.add(new DSpaceCommandLineParameter("-i", "all")); parameters.add(new DSpaceCommandLineParameter("-s", "invalidScope")); @@ -299,7 +209,6 @@ public class CurationScriptIT extends AbstractControllerIntegrationTest { LinkedList parameters = new LinkedList<>(); - parameters.add(new DSpaceCommandLineParameter("-e", admin.getEmail())); parameters.add(new DSpaceCommandLineParameter("-i", "all")); parameters.add(new DSpaceCommandLineParameter("-T", "invalidTaskFile")); @@ -341,7 +250,6 @@ public class CurationScriptIT extends AbstractControllerIntegrationTest { LinkedList parameters = new LinkedList<>(); - parameters.add(new DSpaceCommandLineParameter("-e", admin.getEmail())); parameters.add(new DSpaceCommandLineParameter("-i", publicItem1.getHandle())); parameters.add(new DSpaceCommandLineParameter("-t", CurationClientOptions.getTaskOptions().get(0))); @@ -361,7 +269,7 @@ public class CurationScriptIT extends AbstractControllerIntegrationTest { .andExpect(jsonPath("$", is( ProcessMatcher.matchProcess("curate", String.valueOf(admin.getID()), parameters, - ProcessStatus.SCHEDULED)))) + ProcessStatus.COMPLETED)))) .andDo(result -> idRef .set(read(result.getResponse().getContentAsString(), "$.processId"))); } finally { @@ -394,7 +302,6 @@ public class CurationScriptIT extends AbstractControllerIntegrationTest { File taskFile = new File(testProps.get("test.curateTaskFile").toString()); LinkedList parameters = new LinkedList<>(); - parameters.add(new DSpaceCommandLineParameter("-e", admin.getEmail())); parameters.add(new DSpaceCommandLineParameter("-i", publicItem1.getHandle())); parameters.add(new DSpaceCommandLineParameter("-T", taskFile.getAbsolutePath())); @@ -414,7 +321,7 @@ public class CurationScriptIT extends AbstractControllerIntegrationTest { .andExpect(jsonPath("$", is( ProcessMatcher.matchProcess("curate", String.valueOf(admin.getID()), parameters, - ProcessStatus.SCHEDULED)))) + ProcessStatus.COMPLETED)))) .andDo(result -> idRef .set(read(result.getResponse().getContentAsString(), "$.processId"))); } finally { @@ -422,4 +329,57 @@ public class CurationScriptIT extends AbstractControllerIntegrationTest { } } + @Test + public void curateScript_EPersonInParametersFails() throws Exception { + context.turnOffAuthorisationSystem(); + + String token = getAuthToken(admin.getEmail(), password); + + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Community child1 = CommunityBuilder.createSubCommunity(context, parentCommunity) + .withName("Sub Community") + .build(); + Collection col1 = CollectionBuilder.createCollection(context, child1).withName("Collection 1").build(); + + Item publicItem1 = ItemBuilder.createItem(context, col1) + .withTitle("Public item 1") + .withIssueDate("2017-10-17") + .withAuthor("Smith, Donald").withAuthor("Doe, John") + .withSubject("ExtraEntry") + .build(); + + LinkedList parameters = new LinkedList<>(); + + parameters.add(new DSpaceCommandLineParameter("-e", eperson.getEmail())); + parameters.add(new DSpaceCommandLineParameter("-i", publicItem1.getHandle())); + parameters.add(new DSpaceCommandLineParameter("-t", CurationClientOptions.getTaskOptions().get(0))); + + List list = parameters.stream() + .map(dSpaceCommandLineParameter -> dSpaceRunnableParameterConverter + .convert(dSpaceCommandLineParameter, Projection.DEFAULT)) + .collect(Collectors.toList()); + AtomicReference idRef = new AtomicReference<>(); + + context.restoreAuthSystemState(); + try { + + getClient(token) + .perform(post(CURATE_SCRIPT_ENDPOINT).contentType("multipart/form-data") + .param("properties", + new Gson().toJson(list))) + .andExpect(jsonPath("$", is( + ProcessMatcher.matchProcess("curate", + String.valueOf(admin.getID()), parameters, + ProcessStatus.FAILED)))) + .andDo(result -> idRef + .set(read(result.getResponse().getContentAsString(), "$.processId"))); + } finally { + ProcessBuilder.deleteProcess(idRef.get()); + } + } + + + } diff --git a/dspace/config/spring/api/scripts.xml b/dspace/config/spring/api/scripts.xml index 31d457f09d..cdd2a43d32 100644 --- a/dspace/config/spring/api/scripts.xml +++ b/dspace/config/spring/api/scripts.xml @@ -19,7 +19,7 @@ - + diff --git a/dspace/config/spring/rest/scripts.xml b/dspace/config/spring/rest/scripts.xml index 04cacb4930..d3d7ff437e 100644 --- a/dspace/config/spring/rest/scripts.xml +++ b/dspace/config/spring/rest/scripts.xml @@ -12,4 +12,9 @@ + + + + + \ No newline at end of file From 40ec84d44080976a8ed76d64e9aacde8a0d566a3 Mon Sep 17 00:00:00 2001 From: jonas-atmire Date: Tue, 11 Aug 2020 11:21:50 +0200 Subject: [PATCH 60/96] Remove the ignore annotation from the CurationTest --- dspace-api/src/test/java/org/dspace/curate/CurationTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/dspace-api/src/test/java/org/dspace/curate/CurationTest.java b/dspace-api/src/test/java/org/dspace/curate/CurationTest.java index 184bde0b0c..dadf131c38 100644 --- a/dspace-api/src/test/java/org/dspace/curate/CurationTest.java +++ b/dspace-api/src/test/java/org/dspace/curate/CurationTest.java @@ -18,7 +18,6 @@ import org.dspace.scripts.DSpaceRunnable; import org.dspace.scripts.configuration.ScriptConfiguration; import org.dspace.scripts.factory.ScriptServiceFactory; import org.dspace.scripts.service.ScriptService; -import org.junit.Ignore; import org.junit.Test; public class CurationTest extends AbstractIntegrationTestWithDatabase { @@ -50,7 +49,6 @@ public class CurationTest extends AbstractIntegrationTestWithDatabase { } @Test - @Ignore public void curationWithEPersonParameterTest() throws Exception { context.turnOffAuthorisationSystem(); From 41b862cdd52685ae8c116d7e9b9512a0fc5684c2 Mon Sep 17 00:00:00 2001 From: Raf Ponsaerts Date: Tue, 11 Aug 2020 15:53:39 +0200 Subject: [PATCH 61/96] [Task 72487] addressed feedback regarding builders in CSVMetadataImportReferenceIT and cleared up CsvImportIT to be more descriptive --- .../app/csv/CSVMetadataImportReferenceIT.java | 284 +++++++++--------- .../org/dspace/app/rest/csv/CsvImportIT.java | 9 +- 2 files changed, 143 insertions(+), 150 deletions(-) diff --git a/dspace-api/src/test/java/org/dspace/app/csv/CSVMetadataImportReferenceIT.java b/dspace-api/src/test/java/org/dspace/app/csv/CSVMetadataImportReferenceIT.java index 5ecbd48ba5..2dfe3a781f 100644 --- a/dspace-api/src/test/java/org/dspace/app/csv/CSVMetadataImportReferenceIT.java +++ b/dspace-api/src/test/java/org/dspace/app/csv/CSVMetadataImportReferenceIT.java @@ -19,11 +19,15 @@ import java.util.Iterator; import java.util.List; import java.util.UUID; -import org.dspace.AbstractIntegrationTest; +import org.dspace.AbstractIntegrationTestWithDatabase; import org.dspace.app.bulkedit.MetadataImportException; import org.dspace.app.bulkedit.MetadataImportInvalidHeadingException; import org.dspace.app.scripts.handler.impl.TestDSpaceRunnableHandler; -import org.dspace.authorize.AuthorizeException; +import org.dspace.builder.CollectionBuilder; +import org.dspace.builder.CommunityBuilder; +import org.dspace.builder.EntityTypeBuilder; +import org.dspace.builder.ItemBuilder; +import org.dspace.builder.RelationshipTypeBuilder; import org.dspace.content.Collection; import org.dspace.content.Community; import org.dspace.content.EntityType; @@ -31,23 +35,15 @@ import org.dspace.content.Item; import org.dspace.content.MetadataField; import org.dspace.content.MetadataValue; import org.dspace.content.Relationship; -import org.dspace.content.WorkspaceItem; import org.dspace.content.factory.ContentServiceFactory; -import org.dspace.content.service.CollectionService; -import org.dspace.content.service.CommunityService; -import org.dspace.content.service.EntityTypeService; -import org.dspace.content.service.InstallItemService; import org.dspace.content.service.ItemService; import org.dspace.content.service.MetadataFieldService; import org.dspace.content.service.MetadataValueService; import org.dspace.content.service.RelationshipService; -import org.dspace.content.service.RelationshipTypeService; -import org.dspace.content.service.WorkspaceItemService; import org.dspace.scripts.DSpaceRunnable; import org.dspace.scripts.configuration.ScriptConfiguration; import org.dspace.scripts.factory.ScriptServiceFactory; import org.dspace.scripts.service.ScriptService; -import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -55,20 +51,13 @@ import org.junit.Test; * Created by: Andrew Wood * Date: 26 Jul 2019 */ -public class CSVMetadataImportReferenceIT extends AbstractIntegrationTest { +public class CSVMetadataImportReferenceIT extends AbstractIntegrationTestWithDatabase { //Common collection to utilize for test private Collection col1; private RelationshipService relationshipService = ContentServiceFactory.getInstance().getRelationshipService(); private ItemService itemService = ContentServiceFactory.getInstance().getItemService(); - private CollectionService collectionService = ContentServiceFactory.getInstance().getCollectionService(); - private CommunityService communityService = ContentServiceFactory.getInstance().getCommunityService(); - private WorkspaceItemService workspaceItemService = ContentServiceFactory.getInstance().getWorkspaceItemService(); - private InstallItemService installItemService = ContentServiceFactory.getInstance().getInstallItemService(); - private EntityTypeService entityTypeService = ContentServiceFactory.getInstance().getEntityTypeService(); - private RelationshipTypeService relationshipTypeService = ContentServiceFactory.getInstance() - .getRelationshipTypeService(); Community parentCommunity; @@ -77,72 +66,32 @@ public class CSVMetadataImportReferenceIT extends AbstractIntegrationTest { * Setup testing enviorment */ @Before - public void setup() throws SQLException, AuthorizeException { + public void setup() throws SQLException { context.turnOffAuthorisationSystem(); - parentCommunity = communityService.create(null, context); - communityService.addMetadata(context, parentCommunity, "dc", "title", null, null, "Parent Community"); + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); - col1 = collectionService.create(context, parentCommunity); - collectionService.addMetadata(context, col1, "dc", "title", null, null, "Collection 1"); + col1 = CollectionBuilder.createCollection(context, parentCommunity).withName("Collection 1").build(); - if (entityTypeService.findAll(context).size() > 0) { - //Don't initialize the setup more than once - return; - } context.turnOffAuthorisationSystem(); - EntityType publicationEntityType = entityTypeService.create(context, "Publication"); - EntityType personEntityType = entityTypeService.create(context, "Person"); - EntityType orgUnitType = entityTypeService.create(context, "OrgUnit"); - EntityType projectType = entityTypeService.create(context, "Project"); - relationshipTypeService - .create(context, publicationEntityType, personEntityType, "isAuthorOfPublication", "isPublicationOfAuthor", - null, null, null, null); - relationshipTypeService - .create(context, publicationEntityType, projectType, "isProjectOfPublication", "isPublicationOfProject", 0, - null, 0, null, false, true); + EntityType publication = EntityTypeBuilder.createEntityTypeBuilder(context, "Publication").build(); + EntityType person = EntityTypeBuilder.createEntityTypeBuilder(context, "Person").build(); + EntityType project = EntityTypeBuilder.createEntityTypeBuilder(context, "Project").build(); + EntityType orgUnit = EntityTypeBuilder.createEntityTypeBuilder(context, "OrgUnit").build(); + + RelationshipTypeBuilder + .createRelationshipTypeBuilder(context, publication, person, "isAuthorOfPublication", + "isPublicationOfAuthor", 0, null, 0, + null).withCopyToLeft(false).withCopyToRight(true).build(); + + RelationshipTypeBuilder.createRelationshipTypeBuilder(context, publication, project, "isProjectOfPublication", + "isPublicationOfProject", 0, null, 0, + null).withCopyToRight(true).build(); context.restoreAuthSystemState(); - - context.restoreAuthSystemState(); - } - - @After - @Override - public void destroy() { - context.turnOffAuthorisationSystem(); - try { - List relationships = relationshipService.findAll(context); - for (Relationship relationship : relationships) { - relationshipService.delete(context, relationship); - } - Iterator itemIterator = itemService.findAll(context); - while (itemIterator.hasNext()) { - Item item = itemIterator.next(); - itemService.delete(context, item); - } - List collections = collectionService.findAll(context); - for (Collection collection : collections) { - collectionService.delete(context, collection); - } - List communities = communityService.findAll(context); - for (Community community : communities) { - communityService.delete(context, community); - } - context.commit(); - } catch (Exception e) { - String t = ""; - } - - context.restoreAuthSystemState(); - col1 = null; - parentCommunity = null; - try { - super.destroy(); - } catch (Exception e) { - String t = ""; - } } /** @@ -253,10 +202,14 @@ public class CSVMetadataImportReferenceIT extends AbstractIntegrationTest { @Test public void testSingleUUIDReference() throws Exception { context.turnOffAuthorisationSystem(); - WorkspaceItem workspaceItem = workspaceItemService.create(context, col1, false); - Item person = installItemService.installItem(context, workspaceItem); - itemService.addMetadata(context, person, "relationship", "type", null, null, "Person"); - itemService.update(context, person); + Item person = ItemBuilder.createItem(context, col1) + .withTitle("Author1") + .withIssueDate("2017-10-17") + .withAuthor("Smith, Donald") + .withPersonIdentifierLastName("Smith") + .withPersonIdentifierFirstName("Donald") + .withRelationshipType("Person") + .build(); context.restoreAuthSystemState(); String[] csv = {"id,relationship.type,relation.isAuthorOfPublication,collection,rowName,dc.identifier.other", "+,Publication," + person.getID().toString() + "," + col1.getHandle() + ",anything,0"}; @@ -271,15 +224,22 @@ public class CSVMetadataImportReferenceIT extends AbstractIntegrationTest { @Test public void testMultiUUIDReference() throws Exception { context.turnOffAuthorisationSystem(); - WorkspaceItem workspaceItem = workspaceItemService.create(context, col1, false); - Item person = installItemService.installItem(context, workspaceItem); - itemService.addMetadata(context, person, "relationship", "type", null, null, "Person"); - itemService.update(context, person); - WorkspaceItem workspaceItem2 = workspaceItemService.create(context, col1, false); - Item person2 = installItemService.installItem(context, workspaceItem2); - itemService.addMetadata(context, person2, "relationship", "type", null, null, "Person"); - itemService.update(context, person2); - context.restoreAuthSystemState(); + Item person = ItemBuilder.createItem(context, col1) + .withTitle("Author1") + .withIssueDate("2017-10-17") + .withAuthor("Smith, Donald") + .withPersonIdentifierLastName("Smith") + .withPersonIdentifierFirstName("Donald") + .withRelationshipType("Person") + .build(); + Item person2 = ItemBuilder.createItem(context, col1) + .withTitle("Author2") + .withIssueDate("2017-10-17") + .withAuthor("Smith, John") + .withPersonIdentifierLastName("Smith") + .withPersonIdentifierFirstName("John") + .withRelationshipType("Person") + .build(); String[] csv = {"id,relationship.type,relation.isAuthorOfPublication,collection,rowName,dc.identifier.other", "+,Publication," + person.getID().toString() + "||" + person2.getID().toString() + "," + col1.getHandle() + ",anything,0"}; @@ -295,12 +255,14 @@ public class CSVMetadataImportReferenceIT extends AbstractIntegrationTest { @Test public void testMultiRefArchivedCsv() throws Exception { context.turnOffAuthorisationSystem(); - WorkspaceItem workspaceItem = workspaceItemService.create(context, col1, false); - Item person = installItemService.installItem(context, workspaceItem); - itemService.addMetadata(context, person, "relationship", "type", null, null, "Person"); - itemService.addMetadata(context, person, "dc", "title", null, null, "Person"); - itemService.update(context, person); - + Item person = ItemBuilder.createItem(context, col1) + .withTitle("Person") + .withIssueDate("2017-10-17") + .withAuthor("Smith, Donald") + .withPersonIdentifierLastName("Smith") + .withPersonIdentifierFirstName("Donald") + .withRelationshipType("Person") + .build(); String[] csv = {"id,dc.title,relationship.type,relation.isAuthorOfPublication,collection,rowName," + "dc.identifier.other", "+,Person2,Person,," + col1.getHandle() + ",idVal,0", @@ -319,16 +281,22 @@ public class CSVMetadataImportReferenceIT extends AbstractIntegrationTest { @Test public void testMultiMixedRefArchivedCsv() throws Exception { context.turnOffAuthorisationSystem(); - WorkspaceItem workspaceItem = workspaceItemService.create(context, col1, false); - Item person = installItemService.installItem(context, workspaceItem); - itemService.addMetadata(context, person, "relationship", "type", null, null, "Person"); - itemService.addMetadata(context, person, "dc", "title", null, null, "Person"); - itemService.update(context, person); - WorkspaceItem workspaceItem2 = workspaceItemService.create(context, col1, false); - Item person2 = installItemService.installItem(context, workspaceItem2); - itemService.addMetadata(context, person2, "relationship", "type", null, null, "Person"); - itemService.addMetadata(context, person2, "dc", "title", null, null, "Person2"); - itemService.update(context, person2); + Item person = ItemBuilder.createItem(context, col1) + .withTitle("Person") + .withIssueDate("2017-10-17") + .withAuthor("Smith, Donald") + .withPersonIdentifierLastName("Smith") + .withPersonIdentifierFirstName("Donald") + .withRelationshipType("Person") + .build(); + Item person2 = ItemBuilder.createItem(context, col1) + .withTitle("Person2") + .withIssueDate("2017-10-17") + .withAuthor("Smith, John") + .withPersonIdentifierLastName("Smith") + .withPersonIdentifierFirstName("John") + .withRelationshipType("Person") + .build(); context.restoreAuthSystemState(); String[] csv = {"id,dc.title,relationship.type,relation.isAuthorOfPublication,collection,rowName," + @@ -386,16 +354,24 @@ public class CSVMetadataImportReferenceIT extends AbstractIntegrationTest { @Test(expected = MetadataImportException.class) public void testNonUniqueMDRefInDb() throws Exception { context.turnOffAuthorisationSystem(); - WorkspaceItem workspaceItem = workspaceItemService.create(context, col1, false); - Item person = installItemService.installItem(context, workspaceItem); - itemService.addMetadata(context, person, "relationship", "type", null, null, "Person"); - itemService.addMetadata(context, person, "dc", "identifier", "other", null, "1"); - itemService.update(context, person); - WorkspaceItem workspaceItem2 = workspaceItemService.create(context, col1, false); - Item person2 = installItemService.installItem(context, workspaceItem2); - itemService.addMetadata(context, person2, "relationship", "type", null, null, "Person"); - itemService.addMetadata(context, person2, "dc", "identifier", "other", null, "1"); - itemService.update(context, person2); + Item person = ItemBuilder.createItem(context, col1) + .withTitle("Person") + .withIssueDate("2017-10-17") + .withAuthor("Smith, Donald") + .withPersonIdentifierLastName("Smith") + .withPersonIdentifierFirstName("Donald") + .withRelationshipType("Person") + .withIdentifierOther("1") + .build(); + Item person2 = ItemBuilder.createItem(context, col1) + .withTitle("Person2") + .withIssueDate("2017-10-17") + .withAuthor("Smith, John") + .withPersonIdentifierLastName("Smith") + .withPersonIdentifierFirstName("John") + .withRelationshipType("Person") + .withIdentifierOther("1") + .build(); context.restoreAuthSystemState(); String[] csv = {"id,relationship.type,relation.isAuthorOfPublication,collection,dc.identifier.other", @@ -409,11 +385,15 @@ public class CSVMetadataImportReferenceIT extends AbstractIntegrationTest { @Test(expected = MetadataImportException.class) public void testNonUniqueMDRefInBoth() throws Exception { context.turnOffAuthorisationSystem(); - WorkspaceItem workspaceItem = workspaceItemService.create(context, col1, false); - Item person = installItemService.installItem(context, workspaceItem); - itemService.addMetadata(context, person, "relationship", "type", null, null, "Person"); - itemService.addMetadata(context, person, "dc", "identifier", "other", null, "1"); - itemService.update(context, person); + Item person = ItemBuilder.createItem(context, col1) + .withTitle("Person") + .withIssueDate("2017-10-17") + .withAuthor("Smith, Donald") + .withPersonIdentifierLastName("Smith") + .withPersonIdentifierFirstName("Donald") + .withRelationshipType("Person") + .withIdentifierOther("1") + .build(); context.restoreAuthSystemState(); String[] csv = {"id,relationship.type,relation.isAuthorOfPublication,collection,dc.identifier.other", "+,Person,," + col1.getHandle() + ",1", @@ -471,10 +451,11 @@ public class CSVMetadataImportReferenceIT extends AbstractIntegrationTest { @Test(expected = MetadataImportInvalidHeadingException.class) public void testInvalidRelationshipArchivedOrigin() throws Exception { context.turnOffAuthorisationSystem(); - WorkspaceItem workspaceItem = workspaceItemService.create(context, col1, false); - Item testItem = installItemService.installItem(context, workspaceItem); - itemService.addMetadata(context, testItem, "relationship", "type", null, null, "OrgUnit"); - itemService.update(context, testItem); + Item testItem = ItemBuilder.createItem(context, col1) + .withTitle("OrgUnit") + .withIssueDate("2017-10-17") + .withRelationshipType("OrgUnit") + .build(); context.restoreAuthSystemState(); String[] csv = {"id,relationship.type,relation.isAuthorOfPublication,collection,rowName", "+,Person,," + col1.getHandle() + ",1" + @@ -488,10 +469,11 @@ public class CSVMetadataImportReferenceIT extends AbstractIntegrationTest { @Test(expected = MetadataImportInvalidHeadingException.class) public void testInvalidRelationshipArchivedTarget() throws Exception { context.turnOffAuthorisationSystem(); - WorkspaceItem workspaceItem = workspaceItemService.create(context, col1, false); - Item testItem = installItemService.installItem(context, workspaceItem); - itemService.addMetadata(context, testItem, "relationship", "type", null, null, "OrgUnit"); - itemService.update(context, testItem); + Item testItem = ItemBuilder.createItem(context, col1) + .withTitle("OrgUnit") + .withIssueDate("2017-10-17") + .withRelationshipType("OrgUnit") + .build(); context.restoreAuthSystemState(); String[] csv = {"id,relationship.type,relation.isAuthorOfPublication,collection,rowName", testItem.getID().toString() + ",Person,," + col1.getHandle() + ",1" + @@ -506,25 +488,31 @@ public class CSVMetadataImportReferenceIT extends AbstractIntegrationTest { public void testValidRelationshipNoDefinedTypesInCSV() throws Exception { context.turnOffAuthorisationSystem(); - WorkspaceItem workspaceItem = workspaceItemService.create(context, col1, false); - Item testItem = installItemService.installItem(context, workspaceItem); - itemService.addMetadata(context, testItem, "relationship", "type", null, null, "Person"); - itemService.addMetadata(context, testItem, "dc", "identifier", "other", null, "testItemOne"); - itemService.update(context, testItem); + Item testItem = ItemBuilder.createItem(context, col1) + .withTitle("Person") + .withIssueDate("2017-10-17") + .withAuthor("Smith, Donald") + .withPersonIdentifierLastName("Smith") + .withPersonIdentifierFirstName("Donald") + .withRelationshipType("Person") + .withIdentifierOther("testItemOne") + .build(); - WorkspaceItem workspaceItem2 = workspaceItemService.create(context, col1, false); - Item testItem2 = installItemService.installItem(context, workspaceItem2); - itemService.addMetadata(context, testItem2, "relationship", "type", null, null, "Publication"); - itemService.addMetadata(context, testItem2, "dc", "identifier", "other", null, "testItemTwo"); - itemService.update(context, testItem2); + Item testItem2 = ItemBuilder.createItem(context, col1) + .withTitle("Publication") + .withIssueDate("2017-10-17") + .withRelationshipType("Publication") + .withIdentifierOther("testItemTwo") + .build(); - WorkspaceItem workspaceItem3 = workspaceItemService.create(context, col1, false); - Item testItem3 = installItemService.installItem(context, workspaceItem3); - itemService.addMetadata(context, testItem3, "relationship", "type", null, null, "Project"); - itemService.addMetadata(context, testItem3, "dc", "identifier", "other", null, "testItemThree"); - itemService.update(context, testItem3); + Item testItem3 = ItemBuilder.createItem(context, col1) + .withTitle("Project") + .withIssueDate("2017-10-17") + .withRelationshipType("Project") + .withIdentifierOther("testItemThree") + .build(); context.restoreAuthSystemState(); @@ -558,11 +546,11 @@ public class CSVMetadataImportReferenceIT extends AbstractIntegrationTest { public void testInvalidTypeNameDefined() throws Exception { context.turnOffAuthorisationSystem(); - WorkspaceItem workspaceItem = workspaceItemService.create(context, col1, false); - Item testItem = installItemService.installItem(context, workspaceItem); - itemService.addMetadata(context, testItem, "relationship", "type", null, null, "Publication"); - itemService.update(context, testItem); - + Item testItem = ItemBuilder.createItem(context, col1) + .withTitle("Publication") + .withIssueDate("2017-10-17") + .withRelationshipType("Publication") + .build(); context.restoreAuthSystemState(); String[] csv = {"id,collection,relationship.type,dc.title," + "relation.isProjectOfPublication,relation.isPublicationOfProject", diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CsvImportIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CsvImportIT.java index 9f1f24f83f..7e3e6f6026 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CsvImportIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/csv/CsvImportIT.java @@ -32,6 +32,7 @@ import java.util.stream.Collectors; import com.google.gson.Gson; import org.dspace.app.rest.converter.DSpaceRunnableParameterConverter; +import org.dspace.app.rest.matcher.ProcessMatcher; import org.dspace.app.rest.matcher.RelationshipMatcher; import org.dspace.app.rest.model.ParameterValueRest; import org.dspace.app.rest.projection.Projection; @@ -43,6 +44,7 @@ import org.dspace.builder.ProcessBuilder; import org.dspace.content.Collection; import org.dspace.content.Community; import org.dspace.content.Item; +import org.dspace.content.ProcessStatus; import org.dspace.content.Relationship; import org.dspace.content.service.EntityTypeService; import org.dspace.content.service.ItemService; @@ -276,7 +278,7 @@ public class CsvImportIT extends AbstractEntityIntegrationTest { } @Test - public void createRelationshipsWithCsvImportWithSpecifiedEPersonParameterTest() throws Exception { + public void csvImportWithSpecifiedEPersonParameterTestShouldFailProcess() throws Exception { context.turnOffAuthorisationSystem(); parentCommunity = CommunityBuilder.createCommunity(context) @@ -327,9 +329,12 @@ public class CsvImportIT extends AbstractEntityIntegrationTest { .param("properties", new Gson().toJson(list))) .andExpect(status().isAccepted()) + .andExpect(jsonPath("$", is( + ProcessMatcher.matchProcess("metadata-import", + String.valueOf(admin.getID()), parameters, + ProcessStatus.FAILED)))) .andDo(result -> idRef .set(read(result.getResponse().getContentAsString(), "$.processId"))); - String t = ""; } finally { ProcessBuilder.deleteProcess(idRef.get()); } From cdc1d3b0717a1bdd531a14883e34fe0794437748 Mon Sep 17 00:00:00 2001 From: Raf Ponsaerts Date: Tue, 11 Aug 2020 16:04:03 +0200 Subject: [PATCH 62/96] [Task 72232] added javadoc --- .../src/main/java/org/dspace/curate/Curation.java | 7 +++++++ .../src/main/java/org/dspace/curate/CurationCli.java | 11 +++++++++++ .../dspace/curate/CurationCliScriptConfiguration.java | 4 ++++ .../java/org/dspace/curate/CurationClientOptions.java | 5 +++++ 4 files changed, 27 insertions(+) diff --git a/dspace-api/src/main/java/org/dspace/curate/Curation.java b/dspace-api/src/main/java/org/dspace/curate/Curation.java index 50ad2d1197..44cbb24ed9 100644 --- a/dspace-api/src/main/java/org/dspace/curate/Curation.java +++ b/dspace-api/src/main/java/org/dspace/curate/Curation.java @@ -248,6 +248,13 @@ public class Curation extends DSpaceRunnable { } } + /** + * This method will assign the currentUser to the {@link Context} variable which is also created in this method. + * The instance of the method in this class will fetch the EPersonIdentifier from this class, this identifier + * was given to this class upon instantiation, it'll then be used to find the {@link EPerson} associated with it + * and this {@link EPerson} will be set as the currentUser of the created {@link Context} + * @throws ParseException If something went wrong with the retrieval of the EPerson Identifier + */ protected void assignCurrentUserInContext() throws ParseException { UUID currentUserUuid = this.getEpersonIdentifier(); try { diff --git a/dspace-api/src/main/java/org/dspace/curate/CurationCli.java b/dspace-api/src/main/java/org/dspace/curate/CurationCli.java index cd197b5a16..f70aea5b1d 100644 --- a/dspace-api/src/main/java/org/dspace/curate/CurationCli.java +++ b/dspace-api/src/main/java/org/dspace/curate/CurationCli.java @@ -13,8 +13,19 @@ import org.apache.commons.cli.ParseException; import org.dspace.core.Context; import org.dspace.eperson.EPerson; +/** + * This is the CLI version of the {@link Curation} script. + * This will only be called when the curate script is called from a commandline instance. + */ public class CurationCli extends Curation { + /** + * This is the overridden instance of the {@link Curation#assignCurrentUserInContext()} method in the parent class + * {@link Curation}. + * This is done so that the CLI version of the Script is able to retrieve its currentUser from the -e flag given + * with the parameters of the Script. + * @throws ParseException If the e flag was not given to the parameters when calling the script + */ @Override protected void assignCurrentUserInContext() throws ParseException { if (this.commandLine.hasOption('e')) { diff --git a/dspace-api/src/main/java/org/dspace/curate/CurationCliScriptConfiguration.java b/dspace-api/src/main/java/org/dspace/curate/CurationCliScriptConfiguration.java index 5e1b159bba..5e1d014873 100644 --- a/dspace-api/src/main/java/org/dspace/curate/CurationCliScriptConfiguration.java +++ b/dspace-api/src/main/java/org/dspace/curate/CurationCliScriptConfiguration.java @@ -9,6 +9,10 @@ package org.dspace.curate; import org.apache.commons.cli.Options; +/** + * This is the CLI version of the {@link CurationScriptConfiguration} class that handles the configuration for the + * {@link CurationCli} script + */ public class CurationCliScriptConfiguration extends CurationScriptConfiguration { @Override diff --git a/dspace-api/src/main/java/org/dspace/curate/CurationClientOptions.java b/dspace-api/src/main/java/org/dspace/curate/CurationClientOptions.java index 1db1ac600a..8ec0f14697 100644 --- a/dspace-api/src/main/java/org/dspace/curate/CurationClientOptions.java +++ b/dspace-api/src/main/java/org/dspace/curate/CurationClientOptions.java @@ -45,6 +45,11 @@ public enum CurationClientOptions { return null; } + /** + * This method will create all the possible Options for the {@link Curation} script. + * This will be used by {@link CurationScriptConfiguration} + * @return The options for the {@link Curation} script + */ protected static Options constructOptions() { Options options = new Options(); From ec76371456ca1cf84a649515128b113c6d3d0eaf Mon Sep 17 00:00:00 2001 From: Raf Ponsaerts Date: Wed, 12 Aug 2020 10:31:16 +0200 Subject: [PATCH 63/96] [Task 72430] applied feedback on the filename for MetadataExport --- .../dspace/app/bulkedit/MetadataExport.java | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExport.java b/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExport.java index 195a10e2e2..3332440f06 100644 --- a/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExport.java +++ b/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExport.java @@ -8,13 +8,16 @@ package org.dspace.app.bulkedit; import java.sql.SQLException; -import java.util.UUID; import org.apache.commons.cli.ParseException; +import org.apache.commons.lang3.StringUtils; +import org.dspace.content.DSpaceObject; +import org.dspace.content.factory.ContentServiceFactory; import org.dspace.content.service.MetadataDSpaceCsvExportService; import org.dspace.core.Context; import org.dspace.eperson.factory.EPersonServiceFactory; import org.dspace.eperson.service.EPersonService; +import org.dspace.handle.factory.HandleServiceFactory; import org.dspace.scripts.DSpaceRunnable; import org.dspace.utils.DSpace; @@ -80,17 +83,32 @@ public class MetadataExport extends DSpaceRunnable Date: Tue, 14 Jul 2020 18:17:31 +0200 Subject: [PATCH 64/96] 71752: Metadafield search byFieldName (by solr indexing mdFields) --- .../indexobject/IndexableMetadataField.java | 51 +++++++++++ .../MetadataFieldIndexFactoryImpl.java | 91 +++++++++++++++++++ .../factory/MetadataFieldIndexFactory.java | 19 ++++ .../MetadataFieldRestRepository.java | 72 +++++++++++++++ dspace/config/spring/api/core-services.xml | 1 + 5 files changed, 234 insertions(+) create mode 100644 dspace-api/src/main/java/org/dspace/discovery/indexobject/IndexableMetadataField.java create mode 100644 dspace-api/src/main/java/org/dspace/discovery/indexobject/MetadataFieldIndexFactoryImpl.java create mode 100644 dspace-api/src/main/java/org/dspace/discovery/indexobject/factory/MetadataFieldIndexFactory.java diff --git a/dspace-api/src/main/java/org/dspace/discovery/indexobject/IndexableMetadataField.java b/dspace-api/src/main/java/org/dspace/discovery/indexobject/IndexableMetadataField.java new file mode 100644 index 0000000000..ed44e8eebe --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/discovery/indexobject/IndexableMetadataField.java @@ -0,0 +1,51 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.discovery.indexobject; + +import org.dspace.content.MetadataField; +import org.dspace.discovery.IndexableObject; + +/** + * {@link MetadataField} implementation for the {@link IndexableObject} + * + * @author Maria Verdonck (Atmire) on 14/07/2020 + */ +public class IndexableMetadataField implements IndexableObject { + + private MetadataField metadataField; + public static final String TYPE = MetadataField.class.getSimpleName(); + + public IndexableMetadataField(MetadataField metadataField) { + this.metadataField = metadataField; + } + + @Override + public String getType() { + return TYPE; + } + + @Override + public Integer getID() { + return this.metadataField.getID(); + } + + @Override + public MetadataField getIndexedObject() { + return this.metadataField; + } + + @Override + public void setIndexedObject(MetadataField metadataField) { + this.metadataField = metadataField; + } + + @Override + public String getTypeText() { + return TYPE.toUpperCase(); + } +} diff --git a/dspace-api/src/main/java/org/dspace/discovery/indexobject/MetadataFieldIndexFactoryImpl.java b/dspace-api/src/main/java/org/dspace/discovery/indexobject/MetadataFieldIndexFactoryImpl.java new file mode 100644 index 0000000000..d42b75fcdb --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/discovery/indexobject/MetadataFieldIndexFactoryImpl.java @@ -0,0 +1,91 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.discovery.indexobject; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; + +import org.apache.commons.lang.StringUtils; +import org.apache.solr.common.SolrInputDocument; +import org.dspace.content.MetadataField; +import org.dspace.content.service.MetadataFieldService; +import org.dspace.core.Context; +import org.dspace.discovery.indexobject.factory.MetadataFieldIndexFactory; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Factory implementation for indexing/retrieving {@link org.dspace.content.MetadataField} items in the search core + * + * @author Maria Verdonck (Atmire) on 14/07/2020 + */ +public class MetadataFieldIndexFactoryImpl extends IndexFactoryImpl + implements MetadataFieldIndexFactory { + + @Override + public SolrInputDocument buildDocument(Context context, IndexableMetadataField indexableObject) throws SQLException, + IOException { + // Add the ID's, types and call the SolrServiceIndexPlugins + final SolrInputDocument doc = super.buildDocument(context, indexableObject); + final MetadataField metadataField = indexableObject.getIndexedObject(); + // add schema, element, qualifier and full fieldName + addFacetIndex(doc, "schema", metadataField.getMetadataSchema().getName(), + metadataField.getMetadataSchema().getName()); + addFacetIndex(doc, "element", metadataField.getElement(), metadataField.getElement()); + if (StringUtils.isNotBlank(metadataField.getQualifier())) { + addFacetIndex(doc, "qualifier", metadataField.getQualifier(), metadataField.getQualifier()); + } + String fieldName = metadataField.toString().replace('_', '.'); + addFacetIndex(doc, "fieldName", fieldName, fieldName); + addNamedResourceTypeIndex(doc, indexableObject.getTypeText()); + return doc; + } + + @Autowired + private MetadataFieldService metadataFieldService; + + @Override + public Iterator findAll(Context context) throws SQLException { + final Iterator metadataFields = metadataFieldService.findAll(context).iterator(); + return new Iterator<>() { + @Override + public boolean hasNext() { + return metadataFields.hasNext(); + } + + @Override + public IndexableMetadataField next() { + return new IndexableMetadataField(metadataFields.next()); + } + }; + } + + @Override + public String getType() { + return IndexableMetadataField.TYPE; + } + + @Override + public Optional findIndexableObject(Context context, String id) throws SQLException { + final MetadataField metadataField = metadataFieldService.find(context, Integer.parseInt(id)); + return metadataField == null ? Optional.empty() : Optional.of(new IndexableMetadataField(metadataField)); + } + + @Override + public boolean supports(Object object) { + return object instanceof MetadataField; + } + + @Override + public List getIndexableObjects(Context context, MetadataField object) { + return Arrays.asList(new IndexableMetadataField(object)); + } +} diff --git a/dspace-api/src/main/java/org/dspace/discovery/indexobject/factory/MetadataFieldIndexFactory.java b/dspace-api/src/main/java/org/dspace/discovery/indexobject/factory/MetadataFieldIndexFactory.java new file mode 100644 index 0000000000..976cc4511c --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/discovery/indexobject/factory/MetadataFieldIndexFactory.java @@ -0,0 +1,19 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.discovery.indexobject.factory; + +import org.dspace.content.MetadataField; +import org.dspace.discovery.indexobject.IndexableMetadataField; + +/** + * Factory interface for indexing/retrieving {@link org.dspace.content.MetadataField} items in the search core + * + * @author Maria Verdonck (Atmire) on 14/07/2020 + */ +public interface MetadataFieldIndexFactory extends IndexFactory { +} diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataFieldRestRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataFieldRestRepository.java index b7764b81dc..3834d7b802 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataFieldRestRepository.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataFieldRestRepository.java @@ -9,9 +9,11 @@ package org.dspace.app.rest.repository; import static java.lang.Integer.parseInt; import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.dspace.app.rest.model.SearchConfigurationRest.Filter.OPERATOR_EQUALS; import java.io.IOException; import java.sql.SQLException; +import java.util.ArrayList; import java.util.List; import java.util.Objects; import javax.servlet.http.HttpServletRequest; @@ -19,6 +21,8 @@ import javax.servlet.http.HttpServletRequest; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Logger; import org.dspace.app.rest.Parameter; import org.dspace.app.rest.SearchRestMethod; import org.dspace.app.rest.exception.DSpaceBadRequestException; @@ -31,6 +35,12 @@ import org.dspace.content.NonUniqueMetadataException; import org.dspace.content.service.MetadataFieldService; import org.dspace.content.service.MetadataSchemaService; import org.dspace.core.Context; +import org.dspace.discovery.DiscoverQuery; +import org.dspace.discovery.DiscoverResult; +import org.dspace.discovery.IndexableObject; +import org.dspace.discovery.SearchService; +import org.dspace.discovery.SearchServiceException; +import org.dspace.discovery.indexobject.IndexableMetadataField; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; @@ -45,6 +55,10 @@ import org.springframework.stereotype.Component; */ @Component(MetadataFieldRest.CATEGORY + "." + MetadataFieldRest.NAME) public class MetadataFieldRestRepository extends DSpaceRestRepository { + /** + * log4j logger + */ + private static Logger log = org.apache.logging.log4j.LogManager.getLogger(MetadataFieldRestRepository.class); @Autowired MetadataFieldService metadataFieldService; @@ -52,6 +66,9 @@ public class MetadataFieldRestRepository extends DSpaceRestRepository findByFieldName(@Parameter(value = "schema", required = false) String schemaName, + @Parameter(value = "element", required = false) String elementName, + @Parameter(value = "qualifier", required = false) String qualifierName, + @Parameter(value = "query", required = false) String query, + Pageable pageable) throws SQLException { + Context context = obtainContext(); + List filterQueries = new ArrayList<>(); + if (StringUtils.isNotBlank(schemaName)) { + filterQueries.add(searchService.toFilterQuery(context, "schema", OPERATOR_EQUALS, schemaName).getFilterQuery()); + } + if (StringUtils.isNotBlank(elementName)) { + filterQueries.add(searchService.toFilterQuery(context, "element", OPERATOR_EQUALS, elementName).getFilterQuery()); + } + if (StringUtils.isNotBlank(qualifierName)) { + filterQueries.add(searchService.toFilterQuery(context, "qualifier", OPERATOR_EQUALS, qualifierName).getFilterQuery()); + } + if (StringUtils.isNotBlank(query)) { + filterQueries.add(searchService.toFilterQuery(context, "fieldName", OPERATOR_EQUALS, query).getFilterQuery() + "*"); + } + + DiscoverResult searchResult = null; + DiscoverQuery discoverQuery = new DiscoverQuery(); + discoverQuery.addFilterQueries(filterQueries.toArray(new String[filterQueries.size()])); + + try { + searchResult = searchService.search(context, null, discoverQuery); + } catch (SearchServiceException e) { + log.error("Error while searching with Discovery", e); + throw new IllegalArgumentException("Error while searching with Discovery: " + e.getMessage()); + } + + List matchingMetadataFields = new ArrayList<>(); + for (IndexableObject object: searchResult.getIndexableObjects()) { + if (object instanceof IndexableMetadataField) { + matchingMetadataFields.add(((IndexableMetadataField) object).getIndexedObject()); + } + } + // Correct conversion + return converter.toRestPage(matchingMetadataFields, pageable, utils.obtainProjection()); + } + @Override public Class getDomainClass() { return MetadataFieldRest.class; diff --git a/dspace/config/spring/api/core-services.xml b/dspace/config/spring/api/core-services.xml index 74bdbf85da..7c45609f65 100644 --- a/dspace/config/spring/api/core-services.xml +++ b/dspace/config/spring/api/core-services.xml @@ -126,6 +126,7 @@ + From 036e31c38dc1f4d2ea2997c1eaa40256cd961dc7 Mon Sep 17 00:00:00 2001 From: Marie Verdonck Date: Wed, 15 Jul 2020 12:56:19 +0200 Subject: [PATCH 65/96] 71752: ITs for /metadatafields/search/byFieldName endpoint & indexed MetadataFields now accessible to anon group & checkstyle --- .../MetadataFieldIndexFactoryImpl.java | 8 + .../MetadataFieldRestRepository.java | 16 +- .../rest/MetadatafieldRestRepositoryIT.java | 197 ++++++++++++++++++ 3 files changed, 215 insertions(+), 6 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/discovery/indexobject/MetadataFieldIndexFactoryImpl.java b/dspace-api/src/main/java/org/dspace/discovery/indexobject/MetadataFieldIndexFactoryImpl.java index d42b75fcdb..b402e0cefb 100644 --- a/dspace-api/src/main/java/org/dspace/discovery/indexobject/MetadataFieldIndexFactoryImpl.java +++ b/dspace-api/src/main/java/org/dspace/discovery/indexobject/MetadataFieldIndexFactoryImpl.java @@ -20,6 +20,8 @@ import org.dspace.content.MetadataField; import org.dspace.content.service.MetadataFieldService; import org.dspace.core.Context; import org.dspace.discovery.indexobject.factory.MetadataFieldIndexFactory; +import org.dspace.eperson.Group; +import org.dspace.eperson.service.GroupService; import org.springframework.beans.factory.annotation.Autowired; /** @@ -30,6 +32,9 @@ import org.springframework.beans.factory.annotation.Autowired; public class MetadataFieldIndexFactoryImpl extends IndexFactoryImpl implements MetadataFieldIndexFactory { + @Autowired + protected GroupService groupService; + @Override public SolrInputDocument buildDocument(Context context, IndexableMetadataField indexableObject) throws SQLException, IOException { @@ -46,6 +51,9 @@ public class MetadataFieldIndexFactoryImpl extends IndexFactoryImpl findBySchema(@Parameter(value = "schema", required = true) String schemaName, - Pageable pageable) { + Pageable pageable) { try { Context context = obtainContext(); MetadataSchema schema = metadataSchemaService.find(context, schemaName); @@ -132,16 +132,20 @@ public class MetadataFieldRestRepository extends DSpaceRestRepository filterQueries = new ArrayList<>(); if (StringUtils.isNotBlank(schemaName)) { - filterQueries.add(searchService.toFilterQuery(context, "schema", OPERATOR_EQUALS, schemaName).getFilterQuery()); + filterQueries + .add(searchService.toFilterQuery(context, "schema", OPERATOR_EQUALS, schemaName).getFilterQuery()); } if (StringUtils.isNotBlank(elementName)) { - filterQueries.add(searchService.toFilterQuery(context, "element", OPERATOR_EQUALS, elementName).getFilterQuery()); + filterQueries + .add(searchService.toFilterQuery(context, "element", OPERATOR_EQUALS, elementName).getFilterQuery()); } if (StringUtils.isNotBlank(qualifierName)) { - filterQueries.add(searchService.toFilterQuery(context, "qualifier", OPERATOR_EQUALS, qualifierName).getFilterQuery()); + filterQueries.add( + searchService.toFilterQuery(context, "qualifier", OPERATOR_EQUALS, qualifierName).getFilterQuery()); } if (StringUtils.isNotBlank(query)) { - filterQueries.add(searchService.toFilterQuery(context, "fieldName", OPERATOR_EQUALS, query).getFilterQuery() + "*"); + filterQueries + .add(searchService.toFilterQuery(context, "fieldName", OPERATOR_EQUALS, query).getFilterQuery() + "*"); } DiscoverResult searchResult = null; @@ -156,7 +160,7 @@ public class MetadataFieldRestRepository extends DSpaceRestRepository matchingMetadataFields = new ArrayList<>(); - for (IndexableObject object: searchResult.getIndexableObjects()) { + for (IndexableObject object : searchResult.getIndexableObjects()) { if (object instanceof IndexableMetadataField) { matchingMetadataFields.add(((IndexableMetadataField) object).getIndexedObject()); } diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadatafieldRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadatafieldRestRepositoryIT.java index 5b2bfa6f6d..ab123c5f49 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadatafieldRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadatafieldRestRepositoryIT.java @@ -8,6 +8,7 @@ package org.dspace.app.rest; import static com.jayway.jsonpath.JsonPath.read; +import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -54,6 +55,9 @@ public class MetadatafieldRestRepositoryIT extends AbstractControllerIntegration private MetadataSchema metadataSchema; + private static final String METADATAFIELDS_ENDPOINT = "/api/core/metadatafields/"; + private static final String SEARCH_BYFIELDNAME_ENDPOINT = METADATAFIELDS_ENDPOINT + "search/byFieldName"; + @Autowired private MetadataSchemaService metadataSchemaService; @@ -168,6 +172,199 @@ public class MetadatafieldRestRepositoryIT extends AbstractControllerIntegration .andExpect(status().isBadRequest()); } + @Test + public void findByFieldName_schema() throws Exception { + context.turnOffAuthorisationSystem(); + + MetadataSchema schema = MetadataSchemaBuilder.createMetadataSchema(context, "ASchema", + "http://www.dspace.org/ns/aschema").build(); + + MetadataField metadataField = MetadataFieldBuilder + .createMetadataField(context, schema, "AnElement", "AQualifier", "AScopeNote").build(); + + context.restoreAuthSystemState(); + + super.runDSpaceScript("index-discovery", "-b"); + + getClient().perform(get(SEARCH_BYFIELDNAME_ENDPOINT) + .param("schema", schema.getName())) + .andExpect(status().isOk()) + .andExpect(content().contentType(contentType)) + .andExpect(jsonPath("$._embedded.metadatafields", Matchers.hasItem( + MetadataFieldMatcher.matchMetadataField(metadataField)) + )) + .andExpect(jsonPath("$.page.size", is(20))) + .andExpect(jsonPath("$.page.totalElements", is(1))); + } + + @Test + public void findByFieldName_element() throws Exception { + context.turnOffAuthorisationSystem(); + + MetadataSchema schema = MetadataSchemaBuilder.createMetadataSchema(context, "ASchema", + "http://www.dspace.org/ns/aschema").build(); + MetadataSchema schema2 = MetadataSchemaBuilder.createMetadataSchema(context, "ASchema2", + "http://www.dspace.org/ns/aschema2").build(); + + MetadataField metadataField = MetadataFieldBuilder + .createMetadataField(context, schema, "AnElement", "AQualifier", "AScopeNote").build(); + + MetadataField metadataField2 = MetadataFieldBuilder + .createMetadataField(context, schema2, "AnElement", "AQualifier2", "AScopeNote2").build(); + + context.restoreAuthSystemState(); + + super.runDSpaceScript("index-discovery", "-b"); + + getClient().perform(get(SEARCH_BYFIELDNAME_ENDPOINT) + .param("element", "AnElement")) + .andExpect(status().isOk()) + .andExpect(content().contentType(contentType)) + .andExpect(jsonPath("$._embedded.metadatafields", Matchers.hasItem( + MetadataFieldMatcher.matchMetadataField(metadataField)) + )) + .andExpect(jsonPath("$._embedded.metadatafields", Matchers.hasItem( + MetadataFieldMatcher.matchMetadataField(metadataField2)) + )) + .andExpect(jsonPath("$.page.size", is(20))) + .andExpect(jsonPath("$.page.totalElements", is(2))); + } + + @Test + public void findByFieldName_elementAndQualifier() throws Exception { + context.turnOffAuthorisationSystem(); + + MetadataSchema schema = MetadataSchemaBuilder.createMetadataSchema(context, "ASchema", + "http://www.dspace.org/ns/aschema").build(); + MetadataSchema schema2 = MetadataSchemaBuilder.createMetadataSchema(context, "ASchema2", + "http://www.dspace.org/ns/aschema2").build(); + + MetadataField metadataField = MetadataFieldBuilder + .createMetadataField(context, schema, "AnElement1", "AQualifier", "AScopeNote").build(); + + MetadataField metadataField2 = MetadataFieldBuilder + .createMetadataField(context, schema2, "AnElement2", "AQualifier", "AScopeNote2").build(); + + MetadataField metadataField3 = MetadataFieldBuilder + .createMetadataField(context, schema, "AnElement2", "AQualifier", "AScopeNote2").build(); + + context.restoreAuthSystemState(); + + super.runDSpaceScript("index-discovery", "-b"); + + getClient().perform(get(SEARCH_BYFIELDNAME_ENDPOINT) + .param("element", "AnElement2") + .param("qualifier", "AQualifier")) + .andExpect(status().isOk()) + .andExpect(content().contentType(contentType)) + .andExpect(jsonPath("$._embedded.metadatafields", Matchers.hasItem( + MetadataFieldMatcher.matchMetadataField(metadataField2)) + )) + .andExpect(jsonPath("$._embedded.metadatafields", Matchers.hasItem( + MetadataFieldMatcher.matchMetadataField(metadataField3)) + )) + .andExpect(jsonPath("$._embedded.metadatafields", Matchers.not(hasItem( + MetadataFieldMatcher.matchMetadataField(metadataField)) + ))) + .andExpect(jsonPath("$.page.size", is(20))) + .andExpect(jsonPath("$.page.totalElements", is(2))); + } + + @Test + public void findByFieldName_schemaAndQualifier() throws Exception { + context.turnOffAuthorisationSystem(); + + MetadataSchema schema = MetadataSchemaBuilder.createMetadataSchema(context, "ASchema", + "http://www.dspace.org/ns/aschema").build(); + MetadataSchema schema2 = MetadataSchemaBuilder.createMetadataSchema(context, "ASchema2", + "http://www.dspace.org/ns/aschema2").build(); + + MetadataField metadataField = MetadataFieldBuilder + .createMetadataField(context, schema, "AnElement1", "AQualifier", "AScopeNote").build(); + + MetadataField metadataField2 = MetadataFieldBuilder + .createMetadataField(context, schema2, "AnElement2", "AQualifier", "AScopeNote2").build(); + + MetadataField metadataField3 = MetadataFieldBuilder + .createMetadataField(context, schema, "AnElement3", "AQualifier", "AScopeNote3").build(); + + context.restoreAuthSystemState(); + + super.runDSpaceScript("index-discovery", "-b"); + + getClient().perform(get(SEARCH_BYFIELDNAME_ENDPOINT) + .param("schema", schema.getName()) + .param("qualifier", "AQualifier")) + .andExpect(status().isOk()) + .andExpect(content().contentType(contentType)) + .andExpect(jsonPath("$._embedded.metadatafields", Matchers.hasItem( + MetadataFieldMatcher.matchMetadataField(metadataField)) + )) + .andExpect(jsonPath("$._embedded.metadatafields", Matchers.hasItem( + MetadataFieldMatcher.matchMetadataField(metadataField3)) + )) + .andExpect(jsonPath("$._embedded.metadatafields", Matchers.not(hasItem( + MetadataFieldMatcher.matchMetadataField(metadataField2)) + ))) + .andExpect(jsonPath("$.page.size", is(20))) + .andExpect(jsonPath("$.page.totalElements", is(2))); + } + + @Test + public void findByFieldName_query() throws Exception { + context.turnOffAuthorisationSystem(); + + MetadataSchema schema = MetadataSchemaBuilder.createMetadataSchema(context, "ASchema", + "http://www.dspace.org/ns/aschema").build(); + MetadataSchema schema2 = MetadataSchemaBuilder.createMetadataSchema(context, "ASchema2", + "http://www.dspace.org/ns/aschema2").build(); + + MetadataField metadataField = MetadataFieldBuilder + .createMetadataField(context, schema, "AnElement1", "AQualifier", "AScopeNote").build(); + + MetadataField metadataField2 = MetadataFieldBuilder + .createMetadataField(context, schema2, "AnElement2", "AQualifier", "AScopeNote2").build(); + + MetadataField metadataField3 = MetadataFieldBuilder + .createMetadataField(context, schema, "AnElement3", "AQualifier", "AScopeNote2").build(); + + context.restoreAuthSystemState(); + + super.runDSpaceScript("index-discovery", "-b"); + + getClient().perform(get(SEARCH_BYFIELDNAME_ENDPOINT) + .param("query", schema.getName())) + .andExpect(status().isOk()) + .andExpect(content().contentType(contentType)) + .andExpect(jsonPath("$._embedded.metadatafields", Matchers.hasItem( + MetadataFieldMatcher.matchMetadataField(metadataField)) + )) + .andExpect(jsonPath("$._embedded.metadatafields", Matchers.hasItem( + MetadataFieldMatcher.matchMetadataField(metadataField3)) + )) + .andExpect(jsonPath("$._embedded.metadatafields", Matchers.hasItem( + MetadataFieldMatcher.matchMetadataField(metadataField2)) + )) + .andExpect(jsonPath("$.page.size", is(20))) + .andExpect(jsonPath("$.page.totalElements", is(3))); + + getClient().perform(get(SEARCH_BYFIELDNAME_ENDPOINT) + .param("query", schema.getName() + ".AnElement3")) + .andExpect(status().isOk()) + .andExpect(content().contentType(contentType)) + .andExpect(jsonPath("$._embedded.metadatafields", Matchers.not(hasItem( + MetadataFieldMatcher.matchMetadataField(metadataField)) + ))) + .andExpect(jsonPath("$._embedded.metadatafields", Matchers.hasItem( + MetadataFieldMatcher.matchMetadataField(metadataField3)) + )) + .andExpect(jsonPath("$._embedded.metadatafields", Matchers.not(hasItem( + MetadataFieldMatcher.matchMetadataField(metadataField2)) + ))) + .andExpect(jsonPath("$.page.size", is(20))) + .andExpect(jsonPath("$.page.totalElements", is(1))); + } + @Test public void createSuccess() throws Exception { From 1aacdd2538eae5b0b2a7b4968c518e7a670326fa Mon Sep 17 00:00:00 2001 From: Marie Verdonck Date: Fri, 17 Jul 2020 12:18:46 +0200 Subject: [PATCH 66/96] 71752: endpoint metadatafield/name/{metadata-field-full-name} + tests added & fix for search byFieldName in mdFields with query param starting with element or qualifier --- .../MetadataFieldIndexFactoryImpl.java | 12 +- .../rest/MetadataFieldNameRestController.java | 68 ++++++++++ .../app/rest/model/MetadataFieldRest.java | 1 + .../MetadataFieldRestRepository.java | 14 +- .../MetadataFieldNameRestControllerIT.java | 74 +++++++++++ .../rest/MetadatafieldRestRepositoryIT.java | 122 +++++++++++++----- 6 files changed, 252 insertions(+), 39 deletions(-) create mode 100644 dspace-server-webapp/src/main/java/org/dspace/app/rest/MetadataFieldNameRestController.java create mode 100644 dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadataFieldNameRestControllerIT.java diff --git a/dspace-api/src/main/java/org/dspace/discovery/indexobject/MetadataFieldIndexFactoryImpl.java b/dspace-api/src/main/java/org/dspace/discovery/indexobject/MetadataFieldIndexFactoryImpl.java index b402e0cefb..c020c399ae 100644 --- a/dspace-api/src/main/java/org/dspace/discovery/indexobject/MetadataFieldIndexFactoryImpl.java +++ b/dspace-api/src/main/java/org/dspace/discovery/indexobject/MetadataFieldIndexFactoryImpl.java @@ -14,7 +14,7 @@ import java.util.Iterator; import java.util.List; import java.util.Optional; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.solr.common.SolrInputDocument; import org.dspace.content.MetadataField; import org.dspace.content.service.MetadataFieldService; @@ -45,11 +45,15 @@ public class MetadataFieldIndexFactoryImpl extends IndexFactoryImpl + * endpoint where the metadata-field-full-name parameter can be filled in to match a specific metadata field by name + * There's always at most one metadata field per name. + *

+ * It responds with: + *

+ * The single metadata field if there's a match + * 404 if the metadata field doesn't exist + * + * @author Maria Verdonck (Atmire) on 17/07/2020 + */ +@RestController +@RequestMapping("/api/" + MetadataFieldRest.CATEGORY + "/" + MetadataFieldRest.NAME_PLURAL) +public class MetadataFieldNameRestController { + + @Autowired + private ConverterService converter; + + @Autowired + private MetadataFieldService metadataFieldService; + + @Autowired + private Utils utils; + + @GetMapping("/name/{metadata-field-full-name}") + public MetadataFieldRest get(HttpServletRequest request, HttpServletResponse response, + @PathVariable("metadata-field-full-name") String mdFieldName) { + Context context = ContextUtil.obtainContext(request); + try { + MetadataField metadataField = metadataFieldService.findByString(context, mdFieldName, '.'); + + if (metadataField == null) { + throw new ResourceNotFoundException("There was no metadata field found with name: " + mdFieldName); + } + return converter.toRest(metadataField, utils.obtainProjection()); + } catch (SQLException e) { + throw new RuntimeException(e.getMessage(), e); + } + } +} diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/MetadataFieldRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/MetadataFieldRest.java index 966b3afbbe..4524f82a68 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/MetadataFieldRest.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/MetadataFieldRest.java @@ -18,6 +18,7 @@ import org.dspace.app.rest.RestResourceController; */ public class MetadataFieldRest extends BaseObjectRest { public static final String NAME = "metadatafield"; + public static final String NAME_PLURAL = "metadatafields"; public static final String CATEGORY = RestAddressableModel.CORE; @JsonIgnore diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataFieldRestRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataFieldRestRepository.java index 6a284128fe..ba4a393376 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataFieldRestRepository.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataFieldRestRepository.java @@ -116,7 +116,7 @@ public class MetadataFieldRestRepository extends DSpaceRestRepository filterQueries = new ArrayList<>(); + if (StringUtils.isNotBlank(query)) { + if (query.split("\\.").length > 3) { + throw new IllegalArgumentException("Query param should not contain more than 2 dot (.) separators, " + + "forming schema.element.qualifier metadata field name"); + } + filterQueries + .add(searchService.toFilterQuery(context, "fieldName", OPERATOR_EQUALS, query).getFilterQuery() + "*"); + } if (StringUtils.isNotBlank(schemaName)) { filterQueries .add(searchService.toFilterQuery(context, "schema", OPERATOR_EQUALS, schemaName).getFilterQuery()); @@ -143,10 +151,6 @@ public class MetadataFieldRestRepository extends DSpaceRestRepository idRef.set(read(result.getResponse().getContentAsString(), "$.id"))); @@ -406,9 +468,9 @@ public class MetadatafieldRestRepositoryIT extends AbstractControllerIntegration getClient() .perform(post("/api/core/metadatafields") - .param("schemaId", metadataSchema.getID() + "") - .content(new ObjectMapper().writeValueAsBytes(metadataFieldRest)) - .contentType(contentType)) + .param("schemaId", metadataSchema.getID() + "") + .content(new ObjectMapper().writeValueAsBytes(metadataFieldRest)) + .contentType(contentType)) .andExpect(status().isUnauthorized()); } @@ -424,9 +486,9 @@ public class MetadatafieldRestRepositoryIT extends AbstractControllerIntegration getClient(token) .perform(post("/api/core/metadatafields") - .param("schemaId", metadataSchema.getID() + "") - .content(new ObjectMapper().writeValueAsBytes(metadataFieldRest)) - .contentType(contentType)) + .param("schemaId", metadataSchema.getID() + "") + .content(new ObjectMapper().writeValueAsBytes(metadataFieldRest)) + .contentType(contentType)) .andExpect(status().isForbidden()); } @@ -529,15 +591,15 @@ public class MetadatafieldRestRepositoryIT extends AbstractControllerIntegration getClient(getAuthToken(admin.getEmail(), password)) .perform(put("/api/core/metadatafields/" + metadataField.getID()) - .content(new ObjectMapper().writeValueAsBytes(metadataFieldRest)) - .contentType(contentType)) + .content(new ObjectMapper().writeValueAsBytes(metadataFieldRest)) + .contentType(contentType)) .andExpect(status().isOk()); getClient().perform(get("/api/core/metadatafields/" + metadataField.getID())) .andExpect(status().isOk()) .andExpect(jsonPath("$", MetadataFieldMatcher.matchMetadataFieldByKeys( metadataSchema.getName(), ELEMENT_UPDATED, QUALIFIER_UPDATED) - )); + )); } @Test @@ -557,15 +619,15 @@ public class MetadatafieldRestRepositoryIT extends AbstractControllerIntegration getClient() .perform(put("/api/core/metadatafields/" + metadataField.getID()) - .content(new ObjectMapper().writeValueAsBytes(metadataFieldRest)) - .contentType(contentType)) + .content(new ObjectMapper().writeValueAsBytes(metadataFieldRest)) + .contentType(contentType)) .andExpect(status().isUnauthorized()); getClient().perform(get("/api/core/metadatafields/" + metadataField.getID())) .andExpect(status().isOk()) .andExpect(jsonPath("$", MetadataFieldMatcher.matchMetadataFieldByKeys( metadataSchema.getName(), ELEMENT, QUALIFIER) - )); + )); } @@ -587,15 +649,15 @@ public class MetadatafieldRestRepositoryIT extends AbstractControllerIntegration getClient(getAuthToken(eperson.getEmail(), password)) .perform(put("/api/core/metadatafields/" + metadataField.getID()) - .content(new ObjectMapper().writeValueAsBytes(metadataFieldRest)) - .contentType(contentType)) + .content(new ObjectMapper().writeValueAsBytes(metadataFieldRest)) + .contentType(contentType)) .andExpect(status().isForbidden()); getClient().perform(get("/api/core/metadatafields/" + metadataField.getID())) .andExpect(status().isOk()) .andExpect(jsonPath("$", MetadataFieldMatcher.matchMetadataFieldByKeys( metadataSchema.getName(), ELEMENT, QUALIFIER) - )); + )); } From fdef69b6da02f013eac774a23343cb33a793fe08 Mon Sep 17 00:00:00 2001 From: Marie Verdonck Date: Wed, 22 Jul 2020 17:22:34 +0200 Subject: [PATCH 67/96] 71752: constants for field names; code cleanup --- .../MetadataFieldIndexFactoryImpl.java | 19 +++-- .../MetadataFieldRestRepository.java | 71 ++++++++++++------- 2 files changed, 58 insertions(+), 32 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/discovery/indexobject/MetadataFieldIndexFactoryImpl.java b/dspace-api/src/main/java/org/dspace/discovery/indexobject/MetadataFieldIndexFactoryImpl.java index c020c399ae..4d8b116e0d 100644 --- a/dspace-api/src/main/java/org/dspace/discovery/indexobject/MetadataFieldIndexFactoryImpl.java +++ b/dspace-api/src/main/java/org/dspace/discovery/indexobject/MetadataFieldIndexFactoryImpl.java @@ -32,6 +32,11 @@ import org.springframework.beans.factory.annotation.Autowired; public class MetadataFieldIndexFactoryImpl extends IndexFactoryImpl implements MetadataFieldIndexFactory { + public static final String SCHEMA_FIELD_NAME = "schema"; + public static final String ELEMENT_FIELD_NAME = "element"; + public static final String QUALIFIER_FIELD_NAME = "qualifier"; + public static final String FIELD_NAME_VARIATIONS = "fieldName"; + @Autowired protected GroupService groupService; @@ -42,17 +47,17 @@ public class MetadataFieldIndexFactoryImpl extends IndexFactoryImpl matchingMetadataFields = new ArrayList<>(); + + try { + DiscoverResult searchResult = searchService.search(context, null, discoverQuery); + for (IndexableObject object : searchResult.getIndexableObjects()) { + if (object instanceof IndexableMetadataField) { + matchingMetadataFields.add(((IndexableMetadataField) object).getIndexedObject()); + } + } + } catch (SearchServiceException e) { + log.error("Error while searching with Discovery", e); + throw new IllegalArgumentException("Error while searching with Discovery: " + e.getMessage()); + } + + return converter.toRestPage(matchingMetadataFields, pageable, utils.obtainProjection()); + } + + /** + * Creates a discovery query containing the filter queries derived from the request params + * + * @param context Context request + * @param schemaName an exact match of the prefix of the metadata schema (e.g. "dc", "dcterms", "eperson") + * @param elementName an exact match of the field's element (e.g. "contributor", "title") + * @param qualifierName an exact match of the field's qualifier (e.g. "author", "alternative") + * @param query part of the fully qualified field, should start with the start of the schema, element or + * qualifier (e.g. "dc.ti", "contributor", "auth", "contributor.ot") + * @return Discover query containing the filter queries derived from the request params + * @throws SQLException If DB error + */ + private DiscoverQuery createDiscoverQuery(Context context, String schemaName, String elementName, + String qualifierName, String query) throws SQLException { List filterQueries = new ArrayList<>(); if (StringUtils.isNotBlank(query)) { if (query.split("\\.").length > 3) { throw new IllegalArgumentException("Query param should not contain more than 2 dot (.) separators, " + "forming schema.element.qualifier metadata field name"); } - filterQueries - .add(searchService.toFilterQuery(context, "fieldName", OPERATOR_EQUALS, query).getFilterQuery() + "*"); + filterQueries.add(searchService.toFilterQuery(context, MetadataFieldIndexFactoryImpl.FIELD_NAME_VARIATIONS, + OPERATOR_EQUALS, query).getFilterQuery() + "*"); } if (StringUtils.isNotBlank(schemaName)) { - filterQueries - .add(searchService.toFilterQuery(context, "schema", OPERATOR_EQUALS, schemaName).getFilterQuery()); + filterQueries.add( + searchService.toFilterQuery(context, MetadataFieldIndexFactoryImpl.SCHEMA_FIELD_NAME, OPERATOR_EQUALS, + schemaName).getFilterQuery()); } if (StringUtils.isNotBlank(elementName)) { - filterQueries - .add(searchService.toFilterQuery(context, "element", OPERATOR_EQUALS, elementName).getFilterQuery()); + filterQueries.add( + searchService.toFilterQuery(context, MetadataFieldIndexFactoryImpl.ELEMENT_FIELD_NAME, OPERATOR_EQUALS, + elementName).getFilterQuery()); } if (StringUtils.isNotBlank(qualifierName)) { - filterQueries.add( - searchService.toFilterQuery(context, "qualifier", OPERATOR_EQUALS, qualifierName).getFilterQuery()); + filterQueries.add(searchService + .toFilterQuery(context, MetadataFieldIndexFactoryImpl.QUALIFIER_FIELD_NAME, OPERATOR_EQUALS, + qualifierName).getFilterQuery()); } - DiscoverResult searchResult = null; DiscoverQuery discoverQuery = new DiscoverQuery(); discoverQuery.addFilterQueries(filterQueries.toArray(new String[filterQueries.size()])); - - try { - searchResult = searchService.search(context, null, discoverQuery); - } catch (SearchServiceException e) { - log.error("Error while searching with Discovery", e); - throw new IllegalArgumentException("Error while searching with Discovery: " + e.getMessage()); - } - - List matchingMetadataFields = new ArrayList<>(); - for (IndexableObject object : searchResult.getIndexableObjects()) { - if (object instanceof IndexableMetadataField) { - matchingMetadataFields.add(((IndexableMetadataField) object).getIndexedObject()); - } - } - // Correct conversion - return converter.toRestPage(matchingMetadataFields, pageable, utils.obtainProjection()); + return discoverQuery; } @Override From c046a1c8f3e32630a710f92bb86d225baa262c21 Mon Sep 17 00:00:00 2001 From: Marie Verdonck Date: Thu, 23 Jul 2020 11:30:04 +0200 Subject: [PATCH 68/96] 71752: tests for MetadataFieldIndexFactoryImpl --- .../indexobject/IndexFactoryImpl.java | 5 +- .../MetadataFieldIndexFactoryImpl.java | 7 +- .../MetadataFieldIndexFactoryImplTest.java | 93 +++++++++++++++++++ 3 files changed, 100 insertions(+), 5 deletions(-) create mode 100644 dspace-api/src/test/java/org/dspace/discovery/MetadataFieldIndexFactoryImplTest.java diff --git a/dspace-api/src/main/java/org/dspace/discovery/indexobject/IndexFactoryImpl.java b/dspace-api/src/main/java/org/dspace/discovery/indexobject/IndexFactoryImpl.java index ca1423e593..2e4eb67723 100644 --- a/dspace-api/src/main/java/org/dspace/discovery/indexobject/IndexFactoryImpl.java +++ b/dspace-api/src/main/java/org/dspace/discovery/indexobject/IndexFactoryImpl.java @@ -12,6 +12,7 @@ import java.sql.SQLException; import java.util.Date; import java.util.List; +import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang3.StringUtils; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; @@ -56,7 +57,7 @@ public abstract class IndexFactoryImpl implements doc.addField(SearchUtils.RESOURCE_ID_FIELD, indexableObject.getID().toString()); //Do any additional indexing, depends on the plugins - for (SolrServiceIndexPlugin solrServiceIndexPlugin : solrServiceIndexPlugins) { + for (SolrServiceIndexPlugin solrServiceIndexPlugin : ListUtils.emptyIfNull(solrServiceIndexPlugins)) { solrServiceIndexPlugin.additionalIndex(context, indexableObject, doc); } @@ -190,4 +191,4 @@ public abstract class IndexFactoryImpl implements public void deleteAll() throws IOException, SolrServerException { solrSearchCore.getSolr().deleteByQuery(SearchUtils.RESOURCE_TYPE_FIELD + ":" + getType()); } -} \ No newline at end of file +} diff --git a/dspace-api/src/main/java/org/dspace/discovery/indexobject/MetadataFieldIndexFactoryImpl.java b/dspace-api/src/main/java/org/dspace/discovery/indexobject/MetadataFieldIndexFactoryImpl.java index 4d8b116e0d..518a8ff145 100644 --- a/dspace-api/src/main/java/org/dspace/discovery/indexobject/MetadataFieldIndexFactoryImpl.java +++ b/dspace-api/src/main/java/org/dspace/discovery/indexobject/MetadataFieldIndexFactoryImpl.java @@ -21,6 +21,7 @@ import org.dspace.content.service.MetadataFieldService; import org.dspace.core.Context; import org.dspace.discovery.indexobject.factory.MetadataFieldIndexFactory; import org.dspace.eperson.Group; +import org.dspace.eperson.factory.EPersonServiceFactory; import org.dspace.eperson.service.GroupService; import org.springframework.beans.factory.annotation.Autowired; @@ -37,8 +38,7 @@ public class MetadataFieldIndexFactoryImpl extends IndexFactoryImpl Date: Wed, 29 Jul 2020 16:09:25 +0200 Subject: [PATCH 69/96] 71752: Reindex metadatafields on create/update/delete + tests --- .../content/MetadataFieldServiceImpl.java | 24 +++ .../rest/MetadatafieldRestRepositoryIT.java | 185 ++++++++++++++++-- 2 files changed, 197 insertions(+), 12 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/content/MetadataFieldServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/MetadataFieldServiceImpl.java index c71db2d131..bfd9942736 100644 --- a/dspace-api/src/main/java/org/dspace/content/MetadataFieldServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/MetadataFieldServiceImpl.java @@ -22,6 +22,9 @@ import org.dspace.content.service.MetadataSchemaService; import org.dspace.content.service.MetadataValueService; import org.dspace.core.Context; import org.dspace.core.LogManager; +import org.dspace.discovery.IndexingService; +import org.dspace.discovery.indexobject.IndexableMetadataField; +import org.dspace.services.factory.DSpaceServicesFactory; import org.springframework.beans.factory.annotation.Autowired; /** @@ -46,6 +49,7 @@ public class MetadataFieldServiceImpl implements MetadataFieldService { protected MetadataValueService metadataValueService; @Autowired(required = true) protected MetadataSchemaService metadataSchemaService; + private IndexingService indexer; protected MetadataFieldServiceImpl() { @@ -77,6 +81,8 @@ public class MetadataFieldServiceImpl implements MetadataFieldService { log.info(LogManager.getHeader(context, "create_metadata_field", "metadata_field_id=" + metadataField.getID())); + // Update the index of type metadatafield + this.updateMetadataFieldIndex(context); return metadataField; } @@ -149,6 +155,8 @@ public class MetadataFieldServiceImpl implements MetadataFieldService { "metadata_field_id=" + metadataField.getID() + "element=" + metadataField .getElement() + "qualifier=" + metadataField.getQualifier())); + // Update the index of type metadatafield + this.updateMetadataFieldIndex(context); } @Override @@ -177,6 +185,22 @@ public class MetadataFieldServiceImpl implements MetadataFieldService { log.info(LogManager.getHeader(context, "delete_metadata_field", "metadata_field_id=" + metadataField.getID())); + // Update the index of type metadatafield + this.updateMetadataFieldIndex(context); + } + + /** + * Updates the index of type metadatafield. Indexed metadatafields are used by the /search/byFieldName endpoint, + * see MetadataFieldRestRepository + * @param context DSpace context + */ + private void updateMetadataFieldIndex(Context context) { + if (this.indexer == null) { + this.indexer = DSpaceServicesFactory.getInstance().getServiceManager() + .getServiceByName(IndexingService.class.getName(), + IndexingService.class); + } + this.indexer.updateIndex(context, true, IndexableMetadataField.TYPE); } /** diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadatafieldRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadatafieldRestRepositoryIT.java index de559b0b62..b4d1dfc3bb 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadatafieldRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadatafieldRestRepositoryIT.java @@ -184,8 +184,6 @@ public class MetadatafieldRestRepositoryIT extends AbstractControllerIntegration context.restoreAuthSystemState(); - super.runDSpaceScript("index-discovery", "-b"); - getClient().perform(get(SEARCH_BYFIELDNAME_ENDPOINT) .param("schema", schema.getName())) .andExpect(status().isOk()) @@ -214,8 +212,6 @@ public class MetadatafieldRestRepositoryIT extends AbstractControllerIntegration context.restoreAuthSystemState(); - super.runDSpaceScript("index-discovery", "-b"); - getClient().perform(get(SEARCH_BYFIELDNAME_ENDPOINT) .param("element", "AnElement")) .andExpect(status().isOk()) @@ -250,8 +246,6 @@ public class MetadatafieldRestRepositoryIT extends AbstractControllerIntegration context.restoreAuthSystemState(); - super.runDSpaceScript("index-discovery", "-b"); - getClient().perform(get(SEARCH_BYFIELDNAME_ENDPOINT) .param("element", "AnElement2") .param("qualifier", "AQualifier")) @@ -290,8 +284,6 @@ public class MetadatafieldRestRepositoryIT extends AbstractControllerIntegration context.restoreAuthSystemState(); - super.runDSpaceScript("index-discovery", "-b"); - getClient().perform(get(SEARCH_BYFIELDNAME_ENDPOINT) .param("schema", schema.getName()) .param("qualifier", "AQualifier")) @@ -310,6 +302,45 @@ public class MetadatafieldRestRepositoryIT extends AbstractControllerIntegration .andExpect(jsonPath("$.page.totalElements", is(2))); } + @Test + public void findByFieldName_schemaElementAndQualifier() throws Exception { + context.turnOffAuthorisationSystem(); + + MetadataSchema schema = MetadataSchemaBuilder.createMetadataSchema(context, "ASchema", + "http://www.dspace.org/ns/aschema").build(); + MetadataSchema schema2 = MetadataSchemaBuilder.createMetadataSchema(context, "ASchema2", + "http://www.dspace.org/ns/aschema2").build(); + + MetadataField metadataField = MetadataFieldBuilder + .createMetadataField(context, schema, "AnElement1", "AQualifier", "AScopeNote").build(); + + MetadataField metadataField2 = MetadataFieldBuilder + .createMetadataField(context, schema2, "AnElement2", "AQualifier", "AScopeNote2").build(); + + MetadataField metadataField3 = MetadataFieldBuilder + .createMetadataField(context, schema, "AnElement3", "AQualifier", "AScopeNote3").build(); + + context.restoreAuthSystemState(); + + getClient().perform(get(SEARCH_BYFIELDNAME_ENDPOINT) + .param("schema", schema.getName()) + .param("element", metadataField3.getElement()) + .param("qualifier", metadataField3.getQualifier())) + .andExpect(status().isOk()) + .andExpect(content().contentType(contentType)) + .andExpect(jsonPath("$._embedded.metadatafields", Matchers.not(hasItem( + MetadataFieldMatcher.matchMetadataField(metadataField)) + ))) + .andExpect(jsonPath("$._embedded.metadatafields", Matchers.not(hasItem( + MetadataFieldMatcher.matchMetadataField(metadataField2)) + ))) + .andExpect(jsonPath("$._embedded.metadatafields", Matchers.hasItem( + MetadataFieldMatcher.matchMetadataField(metadataField3)) + )) + .andExpect(jsonPath("$.page.size", is(20))) + .andExpect(jsonPath("$.page.totalElements", is(1))); + } + @Test public void findByFieldName_query() throws Exception { context.turnOffAuthorisationSystem(); @@ -330,8 +361,6 @@ public class MetadatafieldRestRepositoryIT extends AbstractControllerIntegration context.restoreAuthSystemState(); - super.runDSpaceScript("index-discovery", "-b"); - getClient().perform(get(SEARCH_BYFIELDNAME_ENDPOINT) .param("query", schema.getName())) .andExpect(status().isOk()) @@ -401,8 +430,6 @@ public class MetadatafieldRestRepositoryIT extends AbstractControllerIntegration context.restoreAuthSystemState(); - super.runDSpaceScript("index-discovery", "-b"); - getClient().perform(get(SEARCH_BYFIELDNAME_ENDPOINT) .param("query", "test")) .andExpect(status().isOk()) @@ -458,6 +485,49 @@ public class MetadatafieldRestRepositoryIT extends AbstractControllerIntegration } } + @Test + public void create_checkAddedToIndex() throws Exception { + + MetadataFieldRest metadataFieldRest = new MetadataFieldRest(); + metadataFieldRest.setElement("testElementForCreate"); + metadataFieldRest.setQualifier("testQualifierForCreate"); + metadataFieldRest.setScopeNote(SCOPE_NOTE); + + String authToken = getAuthToken(admin.getEmail(), password); + AtomicReference idRef = new AtomicReference<>(); + try { + assertThat(metadataFieldService.findByElement(context, metadataSchema, ELEMENT, QUALIFIER), nullValue()); + + getClient(authToken) + .perform(post("/api/core/metadatafields") + .param("schemaId", metadataSchema.getID() + "") + .param("projection", "full") + .content(new ObjectMapper().writeValueAsBytes(metadataFieldRest)) + .contentType(contentType)) + .andExpect(status().isCreated()) + .andDo(result -> idRef.set(read(result.getResponse().getContentAsString(), "$.id"))); + + getClient(authToken).perform(get("/api/core/metadatafields/" + idRef.get())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", MetadataFieldMatcher.matchMetadataFieldByKeys( + metadataSchema.getName(), "testElementForCreate", "testQualifierForCreate"))); + + // new metadata field found in index + getClient().perform(get(SEARCH_BYFIELDNAME_ENDPOINT) + .param("schema", metadataSchema.getName()) + .param("element", metadataFieldRest.getElement()) + .param("qualifier", metadataFieldRest.getQualifier())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$._embedded.metadatafields", Matchers.hasItem( + MetadataFieldMatcher.matchMetadataFieldByKeys(metadataSchema.getName(), + metadataFieldRest.getElement(), metadataFieldRest.getQualifier())) + )) + .andExpect(jsonPath("$.page.totalElements", is(1))); + } finally { + MetadataFieldBuilder.deleteMetadataField(idRef.get()); + } + } + @Test public void createUnauthorized() throws Exception { @@ -574,6 +644,44 @@ public class MetadatafieldRestRepositoryIT extends AbstractControllerIntegration .andExpect(status().isNotFound()); } + @Test + public void delete_checkDeletedFromIndex() throws Exception { + context.turnOffAuthorisationSystem(); + + MetadataSchema schema = MetadataSchemaBuilder.createMetadataSchema(context, "ASchema", + "http://www.dspace.org/ns/aschema").build(); + + MetadataField metadataField = MetadataFieldBuilder.createMetadataField(context, schema, ELEMENT, QUALIFIER, + SCOPE_NOTE).build(); + + context.restoreAuthSystemState(); + + Integer id = metadataField.getID(); + + getClient().perform(get(SEARCH_BYFIELDNAME_ENDPOINT) + .param("schema", schema.getName()) + .param("element", metadataField.getElement()) + .param("qualifier", metadataField.getQualifier())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$._embedded.metadatafields", Matchers.hasItem( + MetadataFieldMatcher.matchMetadataField(metadataField)) + )); + + getClient(getAuthToken(admin.getEmail(), password)) + .perform(delete("/api/core/metadatafields/" + id)) + .andExpect(status().isNoContent()); + + assertThat(metadataFieldService.find(context, id), nullValue()); + + // deleted metadata field not found in index + getClient().perform(get(SEARCH_BYFIELDNAME_ENDPOINT) + .param("schema", schema.getName()) + .param("element", metadataField.getElement()) + .param("qualifier", metadataField.getQualifier())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.page.totalElements", is(0))); + } + @Test public void update() throws Exception { context.turnOffAuthorisationSystem(); @@ -602,6 +710,59 @@ public class MetadatafieldRestRepositoryIT extends AbstractControllerIntegration )); } + @Test + public void update_checkUpdatedInIndex() throws Exception { + context.turnOffAuthorisationSystem(); + + MetadataField metadataField = MetadataFieldBuilder.createMetadataField(context, ELEMENT, QUALIFIER, SCOPE_NOTE) + .build(); + + getClient().perform(get(SEARCH_BYFIELDNAME_ENDPOINT) + .param("schema", metadataSchema.getName()) + .param("element", metadataField.getElement()) + .param("qualifier", metadataField.getQualifier())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$._embedded.metadatafields", Matchers.hasItem( + MetadataFieldMatcher.matchMetadataFieldByKeys(metadataSchema.getName(), + metadataField.getElement(), metadataField.getQualifier())) + )) + .andExpect(jsonPath("$.page.totalElements", is(1))); + + context.restoreAuthSystemState(); + + MetadataFieldRest metadataFieldRest = new MetadataFieldRest(); + metadataFieldRest.setId(metadataField.getID()); + metadataFieldRest.setElement(ELEMENT_UPDATED); + metadataFieldRest.setQualifier(QUALIFIER_UPDATED); + metadataFieldRest.setScopeNote(SCOPE_NOTE_UPDATED); + + getClient(getAuthToken(admin.getEmail(), password)) + .perform(put("/api/core/metadatafields/" + metadataField.getID()) + .content(new ObjectMapper().writeValueAsBytes(metadataFieldRest)) + .contentType(contentType)) + .andExpect(status().isOk()); + + // new metadata field found in index + getClient().perform(get(SEARCH_BYFIELDNAME_ENDPOINT) + .param("schema", metadataSchema.getName()) + .param("element", ELEMENT_UPDATED) + .param("qualifier", QUALIFIER_UPDATED)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$._embedded.metadatafields", Matchers.hasItem( + MetadataFieldMatcher.matchMetadataFieldByKeys(metadataSchema.getName(), + ELEMENT_UPDATED, QUALIFIER_UPDATED)) + )) + .andExpect(jsonPath("$.page.totalElements", is(1))); + + // original metadata field not found in index + getClient().perform(get(SEARCH_BYFIELDNAME_ENDPOINT) + .param("schema", metadataSchema.getName()) + .param("element", metadataField.getElement()) + .param("qualifier", metadataField.getQualifier())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.page.totalElements", is(0))); + } + @Test public void updateUnauthorized() throws Exception { context.turnOffAuthorisationSystem(); From 75ebc887cc44495d00d24afcef47b819f7110ed7 Mon Sep 17 00:00:00 2001 From: Marie Verdonck Date: Wed, 12 Aug 2020 12:51:42 +0200 Subject: [PATCH 70/96] 71752: /metadatafield/ replace with byfieldName?exactName endpoint + tests --- .../rest/MetadataFieldNameRestController.java | 68 ----------------- .../MetadataFieldRestRepository.java | 59 +++++++++------ .../MetadataFieldNameRestControllerIT.java | 74 ------------------- .../rest/MetadatafieldRestRepositoryIT.java | 64 ++++++++++++++++ 4 files changed, 101 insertions(+), 164 deletions(-) delete mode 100644 dspace-server-webapp/src/main/java/org/dspace/app/rest/MetadataFieldNameRestController.java delete mode 100644 dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadataFieldNameRestControllerIT.java diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/MetadataFieldNameRestController.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/MetadataFieldNameRestController.java deleted file mode 100644 index 35b468ae72..0000000000 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/MetadataFieldNameRestController.java +++ /dev/null @@ -1,68 +0,0 @@ -/** - * The contents of this file are subject to the license and copyright - * detailed in the LICENSE and NOTICE files at the root of the source - * tree and available online at - * - * http://www.dspace.org/license/ - */ -package org.dspace.app.rest; - -import java.sql.SQLException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.dspace.app.rest.converter.ConverterService; -import org.dspace.app.rest.model.MetadataFieldRest; -import org.dspace.app.rest.utils.ContextUtil; -import org.dspace.app.rest.utils.Utils; -import org.dspace.content.MetadataField; -import org.dspace.content.service.MetadataFieldService; -import org.dspace.core.Context; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.rest.webmvc.ResourceNotFoundException; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -/** - * This controller will handle all the incoming calls on the /api/core/metadatafields/name/<:metadata-field-full-name> - * endpoint where the metadata-field-full-name parameter can be filled in to match a specific metadata field by name - * There's always at most one metadata field per name. - *

- * It responds with: - *

- * The single metadata field if there's a match - * 404 if the metadata field doesn't exist - * - * @author Maria Verdonck (Atmire) on 17/07/2020 - */ -@RestController -@RequestMapping("/api/" + MetadataFieldRest.CATEGORY + "/" + MetadataFieldRest.NAME_PLURAL) -public class MetadataFieldNameRestController { - - @Autowired - private ConverterService converter; - - @Autowired - private MetadataFieldService metadataFieldService; - - @Autowired - private Utils utils; - - @GetMapping("/name/{metadata-field-full-name}") - public MetadataFieldRest get(HttpServletRequest request, HttpServletResponse response, - @PathVariable("metadata-field-full-name") String mdFieldName) { - Context context = ContextUtil.obtainContext(request); - try { - MetadataField metadataField = metadataFieldService.findByString(context, mdFieldName, '.'); - - if (metadataField == null) { - throw new ResourceNotFoundException("There was no metadata field found with name: " + mdFieldName); - } - return converter.toRest(metadataField, utils.obtainProjection()); - } catch (SQLException e) { - throw new RuntimeException(e.getMessage(), e); - } - } -} diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataFieldRestRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataFieldRestRepository.java index 25d57671a3..be616e7a54 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataFieldRestRepository.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataFieldRestRepository.java @@ -120,6 +120,9 @@ public class MetadataFieldRestRepository extends DSpaceRestRepository matchingMetadataFields = new ArrayList<>(); - try { - DiscoverResult searchResult = searchService.search(context, null, discoverQuery); - for (IndexableObject object : searchResult.getIndexableObjects()) { - if (object instanceof IndexableMetadataField) { - matchingMetadataFields.add(((IndexableMetadataField) object).getIndexedObject()); + if (StringUtils.isBlank(exactName)) { + // Find matches in Solr Search core + DiscoverQuery discoverQuery = this.createDiscoverQuery(context, schemaName, elementName, qualifierName, query); + try { + DiscoverResult searchResult = searchService.search(context, null, discoverQuery); + for (IndexableObject object : searchResult.getIndexableObjects()) { + if (object instanceof IndexableMetadataField) { + matchingMetadataFields.add(((IndexableMetadataField) object).getIndexedObject()); + } } + } catch (SearchServiceException e) { + log.error("Error while searching with Discovery", e); + throw new IllegalArgumentException("Error while searching with Discovery: " + e.getMessage()); + } + } else { + // Find at most one match with exactName query param in DB + MetadataField exactMatchingMdField = metadataFieldService.findByString(context, exactName, '.'); + if (exactMatchingMdField != null) { + matchingMetadataFields.add(exactMatchingMdField); } - } catch (SearchServiceException e) { - log.error("Error while searching with Discovery", e); - throw new IllegalArgumentException("Error while searching with Discovery: " + e.getMessage()); } return converter.toRestPage(matchingMetadataFields, pageable, utils.obtainProjection()); @@ -202,15 +215,15 @@ public class MetadataFieldRestRepository extends DSpaceRestRepository Date: Wed, 12 Aug 2020 13:07:18 +0200 Subject: [PATCH 71/96] 71752: disallow combination exactName and any of query/schema/element/qualifier query params in /byFieldName search endpoint --- .../MetadataFieldRestRepository.java | 4 + .../rest/MetadatafieldRestRepositoryIT.java | 88 ++++++++++++++----- 2 files changed, 69 insertions(+), 23 deletions(-) diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataFieldRestRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataFieldRestRepository.java index be616e7a54..31ea246efe 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataFieldRestRepository.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataFieldRestRepository.java @@ -153,6 +153,10 @@ public class MetadataFieldRestRepository extends DSpaceRestRepository Date: Wed, 12 Aug 2020 15:37:51 +0200 Subject: [PATCH 72/96] Fix conflict --- .../service/ArXivImportMetadataSourceServiceImpl.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java index 94b31fde96..94e2426fe6 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java @@ -30,6 +30,7 @@ import org.dspace.importer.external.datamodel.ImportRecord; import org.dspace.importer.external.datamodel.Query; import org.dspace.importer.external.exception.MetadataSourceException; import org.dspace.importer.external.service.AbstractImportMetadataSourceService; +import org.dspace.importer.external.service.components.QuerySource; import org.jaxen.JaxenException; /** @@ -38,7 +39,8 @@ import org.jaxen.JaxenException; * @author Pasquale Cavallo (pasquale.cavallo at 4Science dot it) * */ -public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadataSourceService { +public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadataSourceService + implements QuerySource { private WebTarget webTarget; private String baseAddress; @@ -77,7 +79,7 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata * @throws MetadataSourceException if the underlying methods throw any exception. */ @Override - public int getNbRecords(String query) throws MetadataSourceException { + public int getRecordsCount(String query) throws MetadataSourceException { return retry(new CountByQueryCallable(query)); } @@ -90,7 +92,7 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata * @throws MetadataSourceException if the underlying methods throw any exception. */ @Override - public int getNbRecords(Query query) throws MetadataSourceException { + public int getRecordsCount(Query query) throws MetadataSourceException { return retry(new CountByQueryCallable(query)); } @@ -400,5 +402,4 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata public void setBaseAddress(String baseAddress) { this.baseAddress = baseAddress; } - } From 052e775bc532cd81d196b298b11fa855d7051183 Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Wed, 12 Aug 2020 16:48:38 +0200 Subject: [PATCH 73/96] Fix conflict on live import --- .../provider/impl/LiveImportDataProvider.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/external/provider/impl/LiveImportDataProvider.java b/dspace-api/src/main/java/org/dspace/external/provider/impl/LiveImportDataProvider.java index 613c8a4fa7..45855a74ad 100644 --- a/dspace-api/src/main/java/org/dspace/external/provider/impl/LiveImportDataProvider.java +++ b/dspace-api/src/main/java/org/dspace/external/provider/impl/LiveImportDataProvider.java @@ -19,7 +19,7 @@ import org.dspace.external.provider.ExternalDataProvider; import org.dspace.importer.external.datamodel.ImportRecord; import org.dspace.importer.external.exception.MetadataSourceException; import org.dspace.importer.external.metadatamapping.MetadatumDTO; -import org.dspace.importer.external.service.components.MetadataSource; +import org.dspace.importer.external.service.components.QuerySource; /** * This class allows to configure a Live Import Provider as an External Data Provider @@ -29,9 +29,9 @@ import org.dspace.importer.external.service.components.MetadataSource; */ public class LiveImportDataProvider implements ExternalDataProvider { /** - * The {@link MetadataSource} live import provider + * The {@link QuerySource} live import provider */ - private MetadataSource metadataSource; + private QuerySource querySource; /** * An unique human readable identifier for this provider @@ -59,8 +59,8 @@ public class LiveImportDataProvider implements ExternalDataProvider { * This method set the MetadataSource for the ExternalDataProvider * @param metadataSource {@link org.dspace.importer.external.service.components.MetadataSource} implementation used to process the input data */ - public void setMetadataSource(MetadataSource metadataSource) { - this.metadataSource = metadataSource; + public void setMetadataSource(QuerySource querySource) { + this.querySource = querySource; } /** @@ -82,11 +82,11 @@ public class LiveImportDataProvider implements ExternalDataProvider { @Override public Optional getExternalDataObject(String id) { try { - ExternalDataObject externalDataObject = getExternalDataObject(metadataSource.getRecord(id)); + ExternalDataObject externalDataObject = getExternalDataObject(querySource.getRecord(id)); return Optional.of(externalDataObject); } catch (MetadataSourceException e) { throw new RuntimeException( - "The live import provider " + metadataSource.getImportSource() + " throws an exception", e); + "The live import provider " + querySource.getImportSource() + " throws an exception", e); } } @@ -94,11 +94,11 @@ public class LiveImportDataProvider implements ExternalDataProvider { public List searchExternalDataObjects(String query, int start, int limit) { Collection records; try { - records = metadataSource.getRecords(query, start, limit); + records = querySource.getRecords(query, start, limit); return records.stream().map(r -> getExternalDataObject(r)).collect(Collectors.toList()); } catch (MetadataSourceException e) { throw new RuntimeException( - "The live import provider " + metadataSource.getImportSource() + " throws an exception", e); + "The live import provider " + querySource.getImportSource() + " throws an exception", e); } } @@ -110,10 +110,10 @@ public class LiveImportDataProvider implements ExternalDataProvider { @Override public int getNumberOfResults(String query) { try { - return metadataSource.getNbRecords(query); + return querySource.getRecordsCount(query); } catch (MetadataSourceException e) { throw new RuntimeException( - "The live import provider " + metadataSource.getImportSource() + " throws an exception", e); + "The live import provider " + querySource.getImportSource() + " throws an exception", e); } } From 2ef44b15876c7ca9c5e36e96941167a0a3f61db8 Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Thu, 13 Aug 2020 16:35:21 +0200 Subject: [PATCH 74/96] lof exception in SimpleXPathMetadataContributor --- .../contributor/SimpleXpathMetadatumContributor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/SimpleXpathMetadatumContributor.java b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/SimpleXpathMetadatumContributor.java index 91a5fc3fe2..c8d2467d5f 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/SimpleXpathMetadatumContributor.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/SimpleXpathMetadatumContributor.java @@ -166,7 +166,7 @@ public class SimpleXpathMetadatumContributor implements MetadataContributor Date: Thu, 13 Aug 2020 17:01:25 +0200 Subject: [PATCH 75/96] Improve Java on ArXivImportMetadataSourceServiceImpl --- .../ArXivImportMetadataSourceServiceImpl.java | 36 +++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java index 94e2426fe6..6b418423fa 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/service/ArXivImportMetadataSourceServiceImpl.java @@ -46,7 +46,7 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata private String baseAddress; /** - * Find the number of records matching a string query. Supports pagination + * Find the number of records matching the query string in ArXiv. Supports pagination. * * @param query a query string to base the search on. * @param start offset to start at @@ -60,8 +60,11 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata } /** - * Find records based on a object query. - * + * Find records based on a object query and convert them to a list metadata mapped in ImportRecord. + * The entry with the key "query" of the Query's map will be used as query string value. + * + * @see org.dspace.importer.external.datamodel.Query + * @see org.dspace.importer.external.datamodel.ImportRecord * @param query a query object to base the search on. * @return a set of records. Fully transformed. * @throws MetadataSourceException if the underlying methods throw any exception. @@ -72,7 +75,7 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata } /** - * Find the number of records matching a query; + * Find the number of records matching the query string in ArXiv; * * @param query a query object to base the search on. * @return the sum of the matching records over this import source @@ -86,7 +89,9 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata /** * Find the number of records matching a query; - * + * The entry with the key "query" of the Query's map will be used to get the query string. + * + * @see org.dspace.importer.external.datamodel.Query * @param query a query string to base the search on. * @return the sum of the matching records over this import source * @throws MetadataSourceException if the underlying methods throw any exception. @@ -97,7 +102,7 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata } /** - * Get a single record from the source by id + * Get a single record of metadata from the arxiv by ArXiv ID. * * @param id id of the record in ArXiv * @return the first matching record @@ -111,8 +116,10 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata } /** - * Get a single record from the source. - * + * Get a single record from the ArXiv matching the query. + * Field "query" will be used to get data from. + * + * @see org.dspace.importer.external.datamodel.Query * @param query a query matching a single record * @return the first matching record * @throws MetadataSourceException if the underlying methods throw any exception. @@ -169,6 +176,9 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata /** * This class is a Callable implementation to count the number of entries for an ArXiv * query. + * This Callable use as query value to ArXiv the string queryString passed to constructor. + * If the object will be construct through Query.class instance, the value of the Query's + * map with the key "query" will be used. * * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it) * @@ -223,6 +233,9 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata /** * This class is a Callable implementation to get ArXiv entries based on * query object. + * This Callable use as query value the string queryString passed to constructor. + * If the object will be construct through Query.class instance, a Query's map entry with key "query" will be used. + * Pagination is supported too, using the value of the Query's map with keys "start" and "count". * * @see org.dspace.importer.external.datamodel.Query * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it) @@ -273,8 +286,9 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata } /** - * This class is a Callable implementation to get ArXiv entry using ArXiv ID - * + * This class is a Callable implementation to get an ArXiv entry using ArXiv ID + * The ID to use can be passed through the constructor as a String or as Query's map entry, with the key "id". + * * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it) * */ @@ -321,6 +335,8 @@ public class ArXivImportMetadataSourceServiceImpl extends AbstractImportMetadata /** * This class is a Callable implementation to search ArXiv entries * using author and title. + * There are two field in the Query map to pass, with keys "title" and "author" + * (at least one must be used). * * @see org.dspace.importer.external.datamodel.Query * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it) From e1fb87773c4aac18072fb8eef37cf866945d95e4 Mon Sep 17 00:00:00 2001 From: Samuel Date: Thu, 13 Aug 2020 20:11:32 +0200 Subject: [PATCH 76/96] taskid 72455 Cookie Preferences per account --- .../java/org/dspace/eperson/EPersonTest.java | 16 ++++++++++++++++ dspace/config/registries/dspace-types.xml | 14 ++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/dspace-api/src/test/java/org/dspace/eperson/EPersonTest.java b/dspace-api/src/test/java/org/dspace/eperson/EPersonTest.java index 8950bfa409..cd04fbb340 100644 --- a/dspace-api/src/test/java/org/dspace/eperson/EPersonTest.java +++ b/dspace-api/src/test/java/org/dspace/eperson/EPersonTest.java @@ -78,6 +78,22 @@ public class EPersonTest extends AbstractUnitTest { super.destroy(); } + @Test + public void testPreferences() throws Exception { + + ePersonService.addMetadata(context, eperson, "dspace", "cookies", "functional", null, "true"); + ePersonService.addMetadata(context, eperson, "dspace", "cookies", "statistics", null, "false"); + context.commit(); + + assertEquals( + "true", + ePersonService.getMetadataFirstValue(eperson, "dspace", "cookies", "functional", null) + ); + assertEquals( + "false", + ePersonService.getMetadataFirstValue(eperson, "dspace", "cookies", "statistics", null) + ); + } /** * Test of equals method, of class EPerson. diff --git a/dspace/config/registries/dspace-types.xml b/dspace/config/registries/dspace-types.xml index 56985373f0..3c3f460bb8 100644 --- a/dspace/config/registries/dspace-types.xml +++ b/dspace/config/registries/dspace-types.xml @@ -16,5 +16,19 @@ + + dspace + cookies + functional + + + + + dspace + cookies + statistics + + + From c7ecfdbc7b84a9bed8afa71708de9324eb2c351b Mon Sep 17 00:00:00 2001 From: Rob Jenson <5349295+ferthalangur@users.noreply.github.com> Date: Fri, 14 Aug 2020 06:05:17 -0400 Subject: [PATCH 77/96] Update start-handle-server start-handle-server will not start if handle-server terminated abnormally (including system restart). Case error in the line to remove stale lock file. --- dspace/bin/start-handle-server | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspace/bin/start-handle-server b/dspace/bin/start-handle-server index 0df1c3a7f4..b9dd53fbef 100755 --- a/dspace/bin/start-handle-server +++ b/dspace/bin/start-handle-server @@ -37,7 +37,7 @@ if [ "$JAVA_OPTS" = "" ]; then fi # Remove lock file, in case the old Handle server did not shut down properly -rm -f $handledir/txns/lock +rm -f $HANDLEDIR/txns/lock # Start the Handle server, with a special log4j properties file. # We cannot simply write to the same logs, since log4j From 3164be125e8d8d40df5a6b3b332c125150c2af1d Mon Sep 17 00:00:00 2001 From: Marie Verdonck Date: Mon, 17 Aug 2020 11:28:39 +0200 Subject: [PATCH 78/96] 72505: sitemaps config feedback applied & changed base url of sitemap_index files to ui.url --- .../java/org/dspace/app/sitemap/GenerateSitemaps.java | 3 +-- dspace/config/dspace.cfg | 11 +++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java b/dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java index 1729ef92f4..b16630e853 100644 --- a/dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java +++ b/dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java @@ -182,9 +182,8 @@ public class GenerateSitemaps { */ public static void generateSitemaps(boolean makeHTMLMap, boolean makeSitemapOrg) throws SQLException, IOException { String sitemapsEndpoint = configurationService.getProperty("sitemap.path", "sitemaps"); - String sitemapStem = configurationService.getProperty("dspace.server.url") + "/" - + sitemapsEndpoint + "/sitemap"; String uiURLStem = configurationService.getProperty("dspace.ui.url"); + String sitemapStem = uiURLStem + "/" + sitemapsEndpoint + "/sitemap"; File outputDir = new File(configurationService.getProperty("sitemap.dir")); if (!outputDir.exists() && !outputDir.mkdir()) { diff --git a/dspace/config/dspace.cfg b/dspace/config/dspace.cfg index 2e3addd2e1..b443154f2b 100644 --- a/dspace/config/dspace.cfg +++ b/dspace/config/dspace.cfg @@ -1429,8 +1429,9 @@ webui.content_disposition_threshold = 8388608 # the directory where the generated sitemaps are stored sitemap.dir = ${dspace.dir}/sitemaps -# sitemaps endpoint path used in SitemapRestController and in the url paths when sitemaps are generated -sitemap.path = sitemaps +# Customize the path of sitemaps in the server webapp +# Defaults to "sitemaps", which means they are available at ${dspace.server.url}/sitemaps/ +# sitemap.path = sitemaps # # Comma-separated list of search engine URLs to 'ping' when a new Sitemap has @@ -1447,8 +1448,10 @@ sitemap.engineurls = http://www.google.com/webmasters/sitemaps/ping?sitemap= # Define cron for how frequently the sitemap should refresh. # Defaults to running daily at 1:15am -# Remove (comment) out this config to disable the sitemap scheduler -# Syntax is defined at https://www.quartz-scheduler.org/api/2.3.0/org/quartz/CronTrigger.html +# Cron syntax is defined at https://www.quartz-scheduler.org/api/2.3.0/org/quartz/CronTrigger.html +# Remove (comment out) this config to disable the sitemap scheduler. +# Sitemap scheduler can also be disabled by setting to "-" (single dash) in local.cfg. +# Keep in mind, changing the schedule requires rebooting your servlet container, e.g. Tomcat. sitemap.cron = 0 15 1 * * ? ##### SHERPA/Romeo Integration Settings #### From 40e3ae4887d940351237b1fcee98fe7e4e3189e4 Mon Sep 17 00:00:00 2001 From: Raf Ponsaerts Date: Mon, 17 Aug 2020 14:39:13 +0200 Subject: [PATCH 79/96] [Task 72546] created tests that involve the fullprojection for various endpoints --- .../rest/AuthorizationRestRepositoryIT.java | 61 +++++++ .../app/rest/BrowsesResourceControllerIT.java | 165 ++++++++++++++++++ .../app/rest/CollectionRestRepositoryIT.java | 38 ++++ .../app/rest/CommunityRestRepositoryIT.java | 133 ++++++++------ .../dspace/app/rest/ItemRestRepositoryIT.java | 42 +++++ .../rest/WorkflowItemRestRepositoryIT.java | 41 +++++ .../rest/WorkspaceItemRestRepositoryIT.java | 41 +++++ .../app/rest/matcher/CommunityMatcher.java | 30 +++- 8 files changed, 498 insertions(+), 53 deletions(-) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/AuthorizationRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/AuthorizationRestRepositoryIT.java index df2a4c310b..08564852ee 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/AuthorizationRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/AuthorizationRestRepositoryIT.java @@ -1637,6 +1637,65 @@ public class AuthorizationRestRepositoryIT extends AbstractControllerIntegration .andExpect(status().isOk()); } + @Test + public void findByObjectAndFeatureFullProjectionTest() throws Exception { + context.turnOffAuthorisationSystem(); + Community com = CommunityBuilder.createCommunity(context).withName("A test community").build(); + CommunityRest comRest = communityConverter.convert(com, DefaultProjection.DEFAULT); + String comUri = utils.linkToSingleResource(comRest, "self").getHref(); + context.restoreAuthSystemState(); + + String adminToken = getAuthToken(admin.getEmail(), password); + + // verify that it works for administrators - with eperson parameter + getClient(adminToken).perform(get("/api/authz/authorizations/search/object") + .param("uri", comUri) + .param("projection", "full") + .param("feature", alwaysTrue.getName()) + .param("eperson", admin.getID().toString())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.page.totalElements", is(1))) + .andExpect(jsonPath("$._embedded.authorizations", contains( + allOf( + hasJsonPath("$.id", is(admin.getID().toString() + "_" + alwaysTrue.getName() + "_" + + comRest.getUniqueType() + "_" + comRest.getId())), + hasJsonPath("$.type", is("authorization")), + hasJsonPath("$._embedded.feature.id", is(alwaysTrue.getName())), + hasJsonPath("$._embedded.eperson.id", is(admin.getID().toString())), + hasJsonPath("$._embedded.object.id", is(com.getID().toString())) + ) + ))) + // This is the Full Projection data not visible to eperson's full projection + .andExpect(jsonPath("$._embedded.authorizations[0]._embedded.object._embedded.adminGroup", + nullValue())); + + String epersonToken = getAuthToken(eperson.getEmail(), password); + + // verify that it works for administrators - with eperson parameter + getClient(epersonToken).perform(get("/api/authz/authorizations/search/object") + .param("uri", comUri) + .param("projection", "full") + .param("feature", alwaysTrue.getName()) + .param("eperson", eperson.getID().toString())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.page.totalElements", is(1))) + .andExpect(jsonPath("$._embedded.authorizations", contains( + allOf( + hasJsonPath("$.id", + is(eperson.getID().toString() + "_" + alwaysTrue.getName() + "_" + + comRest.getUniqueType() + "_" + comRest.getId())), + hasJsonPath("$.type", is("authorization")), + hasJsonPath("$._embedded.feature.id", is(alwaysTrue.getName())), + hasJsonPath("$._embedded.eperson.id", is(eperson.getID().toString())), + hasJsonPath("$._embedded.object.id", is(com.getID().toString())) + ) + ))) + // This is the Full Projection data not visible to eperson's full projection + .andExpect( + jsonPath("$._embedded.authorizations[0]._embedded.object._embedded.adminGroup") + .doesNotExist()); + } + // utility methods to build authorization ID without having an authorization object private String getAuthorizationID(EPerson eperson, AuthorizationFeature feature, BaseObjectRest obj) { return getAuthorizationID(eperson != null ? eperson.getID().toString() : null, feature.getName(), @@ -1663,4 +1722,6 @@ public class AuthorizationRestRepositoryIT extends AbstractControllerIntegration return (epersonUuid != null ? epersonUuid + "_" : "") + featureName + "_" + type + "_" + id.toString(); } + + } diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/BrowsesResourceControllerIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/BrowsesResourceControllerIT.java index c4abf5240b..31c1c11fca 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/BrowsesResourceControllerIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/BrowsesResourceControllerIT.java @@ -14,6 +14,7 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; @@ -958,4 +959,168 @@ public class BrowsesResourceControllerIT extends AbstractControllerIntegrationTe ))); } + @Test + public void findBrowseByTitleItemsFullProjectionTest() throws Exception { + context.turnOffAuthorisationSystem(); + + //** GIVEN ** + //1. A community-collection structure with one parent community with sub-community and two collections. + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Community child1 = CommunityBuilder.createSubCommunity(context, parentCommunity) + .withName("Sub Community") + .build(); + Collection col1 = CollectionBuilder.createCollection(context, child1).withName("Collection 1").build(); + Collection col2 = CollectionBuilder.createCollection(context, child1).withName("Collection 2").build(); + + //2. Two public items that are readable by Anonymous + Item publicItem1 = ItemBuilder.createItem(context, col1) + .withTitle("Public item 1") + .withIssueDate("2017-10-17") + .withAuthor("Smith, Donald").withAuthor("Doe, John") + .withSubject("Java").withSubject("Unit Testing") + .build(); + + + context.restoreAuthSystemState(); + + getClient().perform(get("/api/discover/browses/title/items") + .param("projection", "full")) + + //** THEN ** + //The status has to be 200 OK + .andExpect(status().isOk()) + //We expect the content type to be "application/hal+json;charset=UTF-8" + .andExpect(content().contentType(contentType)) + // The full projection for anon shouldn't show the adminGroup in the response + .andExpect( + jsonPath("$._embedded.items[0]._embedded.owningCollection._embedded.adminGroup").doesNotExist()); + + + String adminToken = getAuthToken(admin.getEmail(), password); + getClient(adminToken).perform(get("/api/discover/browses/title/items") + .param("projection", "full")) + + //** THEN ** + //The status has to be 200 OK + .andExpect(status().isOk()) + //We expect the content type to be "application/hal+json;charset=UTF-8" + .andExpect(content().contentType(contentType)) + // The full projection for admin should show the adminGroup in the response + .andExpect(jsonPath("$._embedded.items[0]._embedded.owningCollection._embedded.adminGroup", + nullValue())); + } + + @Test + public void browseByAuthorFullProjectionTest() throws Exception { + context.turnOffAuthorisationSystem(); + + //** GIVEN ** + //1. A community-collection structure with one parent community and one collection. + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Collection col1 = CollectionBuilder.createCollection(context, parentCommunity).withName("Collection 1").build(); + + //2. Twenty-one public items that are readable by Anonymous + for (int i = 0; i <= 20; i++) { + ItemBuilder.createItem(context, col1) + .withTitle("Public item " + String.format("%02d", i)) + .withIssueDate("2017-10-17") + .withAuthor("Test, Author" + String.format("%02d", i)) + .withSubject("Java").withSubject("Unit Testing") + .build(); + } + + context.restoreAuthSystemState(); + + + getClient().perform(get("/api/discover/browses/title/items") + .param("projection", "full")) + //** THEN ** + //The status has to be 200 OK + .andExpect(status().isOk()) + //We expect the content type to be "application/hal+json;charset=UTF-8" + .andExpect(content().contentType(contentType)) + //We expect 21 public items + .andExpect(jsonPath("$.page.size", is(20))) + .andExpect(jsonPath("$.page.totalElements", is(21))) + .andExpect(jsonPath("$.page.totalPages", is(2))) + .andExpect(jsonPath("$.page.number", is(0))) + // embedded items are already checked by other test, we focus on links here + .andExpect( + jsonPath("$._links.next.href", Matchers.containsString("/api/discover/browses/title/items?"))) + .andExpect( + jsonPath("$._links.last.href", Matchers.containsString("/api/discover/browses/title/items?"))) + .andExpect(jsonPath("$._links.self.href", Matchers.endsWith("/api/discover/browses/title/items"))) + // The full projection for anon shouldn't show the adminGroup in the response + .andExpect( + jsonPath("$._embedded.items[0]._embedded.owningCollection._embedded.adminGroup").doesNotExist()); + + String adminToken = getAuthToken(admin.getEmail(), password); + getClient(adminToken).perform(get("/api/discover/browses/title/items") + .param("projection", "full")) + //** THEN ** + //The status has to be 200 OK + .andExpect(status().isOk()) + //We expect the content type to be "application/hal+json;charset=UTF-8" + .andExpect(content().contentType(contentType)) + //We expect 21 public items + .andExpect(jsonPath("$.page.size", is(20))) + .andExpect(jsonPath("$.page.totalElements", is(21))) + .andExpect(jsonPath("$.page.totalPages", is(2))) + .andExpect(jsonPath("$.page.number", is(0))) + // embedded items are already checked by other test, we focus on links here + .andExpect(jsonPath("$._links.next.href", + Matchers.containsString("/api/discover/browses/title/items?"))) + .andExpect(jsonPath("$._links.last.href", + Matchers.containsString("/api/discover/browses/title/items?"))) + .andExpect( + jsonPath("$._links.self.href", Matchers.endsWith("/api/discover/browses/title/items"))) + // The full projection for anon shouldn't show the adminGroup in the response + .andExpect(jsonPath("$._embedded.items[0]._embedded.owningCollection._embedded.adminGroup", + nullValue())); + + } + + @Test + public void testBrowseByDateIssuedItemsFullProjectionTest() throws Exception { + context.turnOffAuthorisationSystem(); + + //** GIVEN ** + //1. A community-collection structure with one parent community with sub-community and two collections. + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Community child1 = CommunityBuilder.createSubCommunity(context, parentCommunity) + .withName("Sub Community") + .build(); + Collection col1 = CollectionBuilder.createCollection(context, child1).withName("Collection 1").build(); + Collection col2 = CollectionBuilder.createCollection(context, child1).withName("Collection 2").build(); + + Item item1 = ItemBuilder.createItem(context, col1) + .withTitle("Item 1") + .withIssueDate("2017-10-17") + .build(); + + context.restoreAuthSystemState(); + + getClient().perform(get("/api/discover/browses/dateissued/items") + .param("projection", "full")) + + .andExpect(status().isOk()) + .andExpect(content().contentType(contentType)) + .andExpect( + jsonPath("$._embedded.items[0]._embedded.owningCollection._embedded.adminGroup").doesNotExist()); + + String adminToken = getAuthToken(admin.getEmail(), password); + getClient(adminToken).perform(get("/api/discover/browses/dateissued/items") + .param("projection", "full")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$._embedded.items[0]._embedded.owningCollection._embedded.adminGroup", + nullValue())); + } + + } diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/CollectionRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/CollectionRestRepositoryIT.java index d1522d4c8c..26bead3a02 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/CollectionRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/CollectionRestRepositoryIT.java @@ -305,6 +305,44 @@ public class CollectionRestRepositoryIT extends AbstractControllerIntegrationTes col1.getName(), col1.getID(), col1.getHandle()))); } + @Test + public void findOneCollectionFullProjectionTest() throws Exception { + + //We turn off the authorization system in order to create the structure as defined below + context.turnOffAuthorisationSystem(); + + //** GIVEN ** + //1. A community-collection structure with one parent community with sub-community and one collection. + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Community child1 = CommunityBuilder.createSubCommunity(context, parentCommunity) + .withName("Sub Community") + .build(); + Community child2 = CommunityBuilder.createSubCommunity(context, parentCommunity) + .withName("Sub Community Two") + .build(); + Collection col1 = CollectionBuilder.createCollection(context, child1).withName("Collection 1").build(); + + context.restoreAuthSystemState(); + + + String adminToken = getAuthToken(admin.getEmail(), password); + getClient(adminToken).perform(get("/api/core/collections/" + col1.getID()) + .param("projection", "full")) + .andExpect(status().isOk()) + .andExpect(content().contentType(contentType)) + .andExpect(jsonPath("$", CollectionMatcher.matchCollectionEntryFullProjection( + col1.getName(), col1.getID(), col1.getHandle()))); + + getClient().perform(get("/api/core/collections/" + col1.getID()) + .param("projection", "full")) + .andExpect(status().isOk()) + .andExpect(content().contentType(contentType)) + .andExpect(jsonPath("$", Matchers.not(CollectionMatcher.matchCollectionEntryFullProjection( + col1.getName(), col1.getID(), col1.getHandle())))); + } + @Test public void findOneCollectionUnAuthenticatedTest() throws Exception { diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/CommunityRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/CommunityRestRepositoryIT.java index 862db60489..9d1553ab70 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/CommunityRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/CommunityRestRepositoryIT.java @@ -121,10 +121,10 @@ public class CommunityRestRepositoryIT extends AbstractControllerIntegrationTest getClient(authToken).perform(post("/api/core/communities") .content(mapper.writeValueAsBytes(comm)) .contentType(contentType) - .param("embed", CommunityMatcher.getFullEmbedsParameters())) + .param("embed", CommunityMatcher.getNonAdminEmbeds())) .andExpect(status().isCreated()) .andExpect(content().contentType(contentType)) - .andExpect(jsonPath("$", CommunityMatcher.matchFullEmbeds())) + .andExpect(jsonPath("$", CommunityMatcher.matchNonAdminEmbeds())) .andExpect(jsonPath("$", Matchers.allOf( hasJsonPath("$.id", not(empty())), hasJsonPath("$.uuid", not(empty())), @@ -326,15 +326,15 @@ public class CommunityRestRepositoryIT extends AbstractControllerIntegrationTest context.restoreAuthSystemState(); getClient().perform(get("/api/core/communities") - .param("embed", CommunityMatcher.getFullEmbedsParameters())) + .param("embed", CommunityMatcher.getNonAdminEmbeds())) .andExpect(status().isOk()) .andExpect(content().contentType(contentType)) .andExpect(jsonPath("$._embedded.communities", Matchers.containsInAnyOrder( - CommunityMatcher.matchCommunityEntryFullProjection(parentCommunity.getName(), + CommunityMatcher.matchCommunityEntryNonAdminEmbeds(parentCommunity.getName(), parentCommunity.getID(), parentCommunity.getHandle()), CommunityMatcher - .matchCommunityEntryFullProjection(child1.getName(), child1.getID(), child1.getHandle()) + .matchCommunityEntryNonAdminEmbeds(child1.getName(), child1.getID(), child1.getHandle()) ))) .andExpect(jsonPath("$._links.self.href", Matchers.containsString("/api/core/communities"))) .andExpect(jsonPath("$.page.size", is(20))) @@ -360,13 +360,13 @@ public class CommunityRestRepositoryIT extends AbstractControllerIntegrationTest context.restoreAuthSystemState(); getClient().perform(get("/api/core/communities").param("size", "2") - .param("embed", CommunityMatcher.getFullEmbedsParameters())) + .param("embed", CommunityMatcher.getNonAdminEmbeds())) .andExpect(status().isOk()) .andExpect(content().contentType(contentType)) .andExpect(jsonPath("$._embedded.communities", Matchers.containsInAnyOrder( CommunityMatcher.matchCommunityEntryMultipleTitles(titles, parentCommunity.getID(), parentCommunity.getHandle()), - CommunityMatcher.matchCommunityEntryFullProjection(child1.getName(), child1.getID(), + CommunityMatcher.matchCommunityEntryNonAdminEmbeds(child1.getName(), child1.getID(), child1.getHandle()) ))) .andExpect(jsonPath("$._links.self.href", Matchers.containsString("/api/core/communities"))) @@ -392,13 +392,13 @@ public class CommunityRestRepositoryIT extends AbstractControllerIntegrationTest context.restoreAuthSystemState(); getClient().perform(get("/api/core/communities").param("size", "2") - .param("embed", CommunityMatcher.getFullEmbedsParameters())) + .param("embed", CommunityMatcher.getNonAdminEmbeds())) .andExpect(status().isOk()) .andExpect(content().contentType(contentType)) .andExpect(jsonPath("$._embedded.communities", Matchers.containsInAnyOrder( CommunityMatcher.matchCommunityEntryMultipleTitles(titles, parentCommunity.getID(), parentCommunity.getHandle()), - CommunityMatcher.matchCommunityEntryFullProjection(childCommunity.getName(), + CommunityMatcher.matchCommunityEntryNonAdminEmbeds(childCommunity.getName(), childCommunity.getID(), childCommunity.getHandle()) ))) @@ -408,14 +408,14 @@ public class CommunityRestRepositoryIT extends AbstractControllerIntegrationTest 2, 4))); getClient().perform(get("/api/core/communities").param("size", "2").param("page", "1") - .param("embed", CommunityMatcher.getFullEmbedsParameters())) + .param("embed", CommunityMatcher.getNonAdminEmbeds())) .andExpect(status().isOk()) .andExpect(content().contentType(contentType)) .andExpect(jsonPath("$._embedded.communities", Matchers.containsInAnyOrder( - CommunityMatcher.matchCommunityEntryFullProjection(secondParentCommunity.getName(), + CommunityMatcher.matchCommunityEntryNonAdminEmbeds(secondParentCommunity.getName(), secondParentCommunity.getID(), secondParentCommunity.getHandle()), - CommunityMatcher.matchCommunityEntryFullProjection(thirdParentCommunity.getName(), + CommunityMatcher.matchCommunityEntryNonAdminEmbeds(thirdParentCommunity.getName(), thirdParentCommunity.getID(), thirdParentCommunity.getHandle()) ))) @@ -433,11 +433,11 @@ public class CommunityRestRepositoryIT extends AbstractControllerIntegrationTest context.restoreAuthSystemState(); getClient().perform(get("/api/core/communities") - .param("embed", CommunityMatcher.getFullEmbedsParameters())) + .param("embed", CommunityMatcher.getNonAdminEmbeds())) .andExpect(status().isOk()) .andExpect(content().contentType(contentType)) .andExpect(jsonPath("$._embedded.communities", Matchers.contains( - CommunityMatcher.matchCommunityEntryFullProjection(parentCommunity.getName(), + CommunityMatcher.matchCommunityEntryNonAdminEmbeds(parentCommunity.getName(), parentCommunity.getID(), parentCommunity.getHandle()) ))) @@ -499,17 +499,17 @@ public class CommunityRestRepositoryIT extends AbstractControllerIntegrationTest getClient().perform(get("/api/core/communities") .param("size", "1") - .param("embed", CommunityMatcher.getFullEmbedsParameters())) + .param("embed", CommunityMatcher.getNonAdminEmbeds())) .andExpect(status().isOk()) .andExpect(content().contentType(contentType)) .andExpect(jsonPath("$._embedded.communities", Matchers.contains( - CommunityMatcher.matchCommunityEntryFullProjection(parentCommunity.getName(), + CommunityMatcher.matchCommunityEntryNonAdminEmbeds(parentCommunity.getName(), parentCommunity.getID(), parentCommunity.getHandle()) ))) .andExpect(jsonPath("$._embedded.communities", Matchers.not( Matchers.contains( - CommunityMatcher.matchCommunityEntryFullProjection(child1.getName(), child1.getID(), + CommunityMatcher.matchCommunityEntryNonAdminEmbeds(child1.getName(), child1.getID(), child1.getHandle()) ) ))) @@ -519,16 +519,16 @@ public class CommunityRestRepositoryIT extends AbstractControllerIntegrationTest getClient().perform(get("/api/core/communities") .param("size", "1") .param("page", "1") - .param("embed", CommunityMatcher.getFullEmbedsParameters())) + .param("embed", CommunityMatcher.getNonAdminEmbeds())) .andExpect(status().isOk()) .andExpect(content().contentType(contentType)) .andExpect(jsonPath("$._embedded.communities", Matchers.contains( - CommunityMatcher.matchCommunityEntryFullProjection(child1.getName(), child1.getID(), + CommunityMatcher.matchCommunityEntryNonAdminEmbeds(child1.getName(), child1.getID(), child1.getHandle()) ))) .andExpect(jsonPath("$._embedded.communities", Matchers.not( Matchers.contains( - CommunityMatcher.matchCommunityEntryFullProjection(parentCommunity.getName(), + CommunityMatcher.matchCommunityEntryNonAdminEmbeds(parentCommunity.getName(), parentCommunity.getID(), parentCommunity.getHandle()) ) @@ -662,10 +662,10 @@ public class CommunityRestRepositoryIT extends AbstractControllerIntegrationTest // When full projection is requested, response should include expected properties, links, and embeds. getClient().perform(get("/api/core/communities/" + parentCommunity.getID().toString()) - .param("embed", CommunityMatcher.getFullEmbedsParameters())) + .param("embed", CommunityMatcher.getNonAdminEmbeds())) .andExpect(status().isOk()) .andExpect(content().contentType(contentType)) - .andExpect(jsonPath("$", CommunityMatcher.matchFullEmbeds())) + .andExpect(jsonPath("$", CommunityMatcher.matchNonAdminEmbeds())) .andExpect(jsonPath("$", CommunityMatcher.matchCommunityEntry( parentCommunity.getName(), parentCommunity.getID(), parentCommunity.getHandle()))); @@ -679,6 +679,39 @@ public class CommunityRestRepositoryIT extends AbstractControllerIntegrationTest parentCommunity.getName(), parentCommunity.getID(), parentCommunity.getHandle()))); } + @Test + public void findOneFullProjectionTest() throws Exception { + //We turn off the authorization system in order to create the structure as defined below + context.turnOffAuthorisationSystem(); + + //** GIVEN ** + //1. A community-collection structure with one parent community with sub-community and one collection. + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Community child1 = CommunityBuilder.createSubCommunity(context, parentCommunity) + .withName("Sub Community") + .build(); + Collection col1 = CollectionBuilder.createCollection(context, child1).withName("Collection 1").build(); + + context.restoreAuthSystemState(); + + String adminToken = getAuthToken(admin.getEmail(), password); + getClient(adminToken).perform(get("/api/core/communities/" + parentCommunity.getID().toString()) + .param("projection", "full")) + .andExpect(status().isOk()) + .andExpect(content().contentType(contentType)) + .andExpect(jsonPath("$", CommunityMatcher.matchCommunityEntryFullProjection( + parentCommunity.getName(), parentCommunity.getID(), parentCommunity.getHandle()))); + + getClient().perform(get("/api/core/communities/" + parentCommunity.getID().toString()) + .param("projection", "full")) + .andExpect(status().isOk()) + .andExpect(content().contentType(contentType)) + .andExpect(jsonPath("$", Matchers.not(CommunityMatcher.matchCommunityEntryFullProjection( + parentCommunity.getName(), parentCommunity.getID(), parentCommunity.getHandle())))); + } + @Test public void findOneUnAuthenticatedTest() throws Exception { context.turnOffAuthorisationSystem(); @@ -778,17 +811,17 @@ public class CommunityRestRepositoryIT extends AbstractControllerIntegrationTest context.restoreAuthSystemState(); getClient().perform(get("/api/core/communities/" + parentCommunity.getID().toString()) - .param("embed", CommunityMatcher.getFullEmbedsParameters())) + .param("embed", CommunityMatcher.getNonAdminEmbeds())) .andExpect(status().isOk()) .andExpect(content().contentType(contentType)) .andExpect(jsonPath("$", Matchers.is( - CommunityMatcher.matchCommunityEntryFullProjection(parentCommunity.getName(), + CommunityMatcher.matchCommunityEntryNonAdminEmbeds(parentCommunity.getName(), parentCommunity.getID(), parentCommunity.getHandle()) ))) .andExpect(jsonPath("$", Matchers.not( Matchers.is( - CommunityMatcher.matchCommunityEntryFullProjection(child1.getName(), child1.getID(), + CommunityMatcher.matchCommunityEntryNonAdminEmbeds(child1.getName(), child1.getID(), child1.getHandle()) ) ))) @@ -860,21 +893,21 @@ public class CommunityRestRepositoryIT extends AbstractControllerIntegrationTest context.restoreAuthSystemState(); getClient().perform(get("/api/core/communities/search/top") - .param("embed", CommunityMatcher.getFullEmbedsParameters())) + .param("embed", CommunityMatcher.getNonAdminEmbeds())) .andExpect(status().isOk()) .andExpect(content().contentType(contentType)) .andExpect(jsonPath("$._embedded.communities", Matchers.containsInAnyOrder( - CommunityMatcher.matchCommunityEntryFullProjection(parentCommunity.getName(), + CommunityMatcher.matchCommunityEntryNonAdminEmbeds(parentCommunity.getName(), parentCommunity.getID(), parentCommunity.getHandle()), - CommunityMatcher.matchCommunityEntryFullProjection(parentCommunity2.getName(), + CommunityMatcher.matchCommunityEntryNonAdminEmbeds(parentCommunity2.getName(), parentCommunity2.getID(), parentCommunity2.getHandle()) ))) .andExpect(jsonPath("$._embedded.communities", Matchers.not(Matchers.containsInAnyOrder( - CommunityMatcher.matchCommunityEntryFullProjection(child1.getName(), child1.getID(), + CommunityMatcher.matchCommunityEntryNonAdminEmbeds(child1.getName(), child1.getID(), child1.getHandle()), - CommunityMatcher.matchCommunityEntryFullProjection(child12.getName(), child12.getID(), + CommunityMatcher.matchCommunityEntryNonAdminEmbeds(child12.getName(), child12.getID(), child12.getHandle()) )))) .andExpect( @@ -1337,17 +1370,17 @@ public class CommunityRestRepositoryIT extends AbstractControllerIntegrationTest context.restoreAuthSystemState(); getClient().perform(get("/api/core/communities/" + parentCommunity.getID().toString()) - .param("embed", CommunityMatcher.getFullEmbedsParameters())) + .param("embed", CommunityMatcher.getNonAdminEmbeds())) .andExpect(status().isOk()) .andExpect(content().contentType(contentType)) .andExpect(jsonPath("$", Matchers.is( - CommunityMatcher.matchCommunityEntryFullProjection(parentCommunity.getName(), + CommunityMatcher.matchCommunityEntryNonAdminEmbeds(parentCommunity.getName(), parentCommunity.getID(), parentCommunity.getHandle()) ))) .andExpect(jsonPath("$", Matchers.not( Matchers.is( - CommunityMatcher.matchCommunityEntryFullProjection(child1.getName(), child1.getID(), + CommunityMatcher.matchCommunityEntryNonAdminEmbeds(child1.getName(), child1.getID(), child1.getHandle()) ) ))) @@ -1374,13 +1407,13 @@ public class CommunityRestRepositoryIT extends AbstractControllerIntegrationTest ; getClient().perform(get("/api/core/communities/" + parentCommunity.getID().toString()) - .param("embed", CommunityMatcher.getFullEmbedsParameters())) + .param("embed", CommunityMatcher.getNonAdminEmbeds())) .andExpect(status().isOk()) .andExpect(content().contentType(contentType)) .andExpect(jsonPath("$", Matchers.is( - CommunityMatcher.matchCommunityEntryFullProjection("Electronic theses and dissertations", - parentCommunity.getID(), - parentCommunity.getHandle()) + CommunityMatcher.matchCommunityEntryNonAdminEmbeds("Electronic theses and dissertations", + parentCommunity.getID(), + parentCommunity.getHandle()) ))) .andExpect(jsonPath("$._links.self.href", Matchers.containsString("/api/core/communities"))) ; @@ -1429,11 +1462,11 @@ public class CommunityRestRepositoryIT extends AbstractControllerIntegrationTest context.restoreAuthSystemState(); getClient(token).perform(get("/api/core/communities/" + parentCommunity.getID().toString()) - .param("embed", CommunityMatcher.getFullEmbedsParameters())) + .param("embed", CommunityMatcher.getNonAdminEmbeds())) .andExpect(status().isOk()) .andExpect(content().contentType(contentType)) .andExpect(jsonPath("$", Matchers.is( - CommunityMatcher.matchCommunityEntryFullProjection(parentCommunity.getName(), + CommunityMatcher.matchCommunityEntryNonAdminEmbeds(parentCommunity.getName(), parentCommunity.getID(), parentCommunity.getHandle()) ))) @@ -1492,11 +1525,11 @@ public class CommunityRestRepositoryIT extends AbstractControllerIntegrationTest context.restoreAuthSystemState(); getClient().perform(get("/api/core/communities/" + parentCommunity.getID().toString()) - .param("embed", CommunityMatcher.getFullEmbedsParameters())) + .param("embed", CommunityMatcher.getNonAdminEmbeds())) .andExpect(status().isOk()) .andExpect(content().contentType(contentType)) .andExpect(jsonPath("$", Matchers.is( - CommunityMatcher.matchCommunityEntryFullProjection(parentCommunity.getName(), + CommunityMatcher.matchCommunityEntryNonAdminEmbeds(parentCommunity.getName(), parentCommunity.getID(), parentCommunity.getHandle()) ))) @@ -1526,11 +1559,11 @@ public class CommunityRestRepositoryIT extends AbstractControllerIntegrationTest context.restoreAuthSystemState(); getClient(token).perform(get("/api/core/communities/" + parentCommunity.getID().toString()) - .param("embed", CommunityMatcher.getFullEmbedsParameters())) + .param("embed", CommunityMatcher.getNonAdminEmbeds())) .andExpect(status().isOk()) .andExpect(content().contentType(contentType)) .andExpect(jsonPath("$", Matchers.is( - CommunityMatcher.matchCommunityEntryFullProjection(parentCommunity.getName(), + CommunityMatcher.matchCommunityEntryNonAdminEmbeds(parentCommunity.getName(), parentCommunity.getID(), parentCommunity.getHandle()) ))) @@ -1563,17 +1596,17 @@ public class CommunityRestRepositoryIT extends AbstractControllerIntegrationTest context.restoreAuthSystemState(); getClient().perform(get("/api/core/communities/" + parentCommunity.getID().toString()) - .param("embed", CommunityMatcher.getFullEmbedsParameters())) + .param("embed", CommunityMatcher.getNonAdminEmbeds())) .andExpect(status().isOk()) .andExpect(content().contentType(contentType)) .andExpect(jsonPath("$", Matchers.is( - CommunityMatcher.matchCommunityEntryFullProjection(parentCommunity.getName(), + CommunityMatcher.matchCommunityEntryNonAdminEmbeds(parentCommunity.getName(), parentCommunity.getID(), parentCommunity.getHandle()) ))) .andExpect(jsonPath("$", Matchers.not( Matchers.is( - CommunityMatcher.matchCommunityEntryFullProjection(child1.getName(), child1.getID(), + CommunityMatcher.matchCommunityEntryNonAdminEmbeds(child1.getName(), child1.getID(), child1.getHandle()) ) ))) @@ -1603,13 +1636,13 @@ public class CommunityRestRepositoryIT extends AbstractControllerIntegrationTest ; getClient().perform(get("/api/core/communities/" + parentCommunity.getID().toString()) - .param("embed", CommunityMatcher.getFullEmbedsParameters())) + .param("embed", CommunityMatcher.getNonAdminEmbeds())) .andExpect(status().isOk()) .andExpect(content().contentType(contentType)) .andExpect(jsonPath("$", Matchers.is( - CommunityMatcher.matchCommunityEntryFullProjection("Electronic theses and dissertations", - parentCommunity.getID(), - parentCommunity.getHandle()) + CommunityMatcher.matchCommunityEntryNonAdminEmbeds("Electronic theses and dissertations", + parentCommunity.getID(), + parentCommunity.getHandle()) ))) .andExpect(jsonPath("$._links.self.href", Matchers.containsString("/api/core/communities"))) ; diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ItemRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ItemRestRepositoryIT.java index 37b26d5c14..966441c427 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ItemRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ItemRestRepositoryIT.java @@ -13,6 +13,7 @@ import static org.dspace.app.rest.matcher.MetadataMatcher.matchMetadata; import static org.dspace.app.rest.matcher.MetadataMatcher.matchMetadataDoesNotExist; import static org.dspace.core.Constants.WRITE; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.patch; @@ -288,6 +289,47 @@ public class ItemRestRepositoryIT extends AbstractControllerIntegrationTest { .andExpect(jsonPath("$", publicItem1Matcher)); } + @Test + public void findOneFullProjectionTest() throws Exception { + context.turnOffAuthorisationSystem(); + + //** GIVEN ** + //1. A community-collection structure with one parent community with sub-community and two collections. + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Community child1 = CommunityBuilder.createSubCommunity(context, parentCommunity) + .withName("Sub Community") + .build(); + Collection col1 = CollectionBuilder.createCollection(context, child1).withName("Collection 1").build(); + Collection col2 = CollectionBuilder.createCollection(context, child1).withName("Collection 2").build(); + + //2. Three public items that are readable by Anonymous with different subjects + Item publicItem1 = ItemBuilder.createItem(context, col1) + .withTitle("Public item 1") + .withIssueDate("2017-10-17") + .withAuthor("Smith, Donald").withAuthor("Doe, John") + .withSubject("ExtraEntry") + .build(); + context.restoreAuthSystemState(); + Matcher publicItem1Matcher = ItemMatcher.matchItemWithTitleAndDateIssued(publicItem1, + "Public item 1", + "2017-10-17"); + + String token = getAuthToken(admin.getEmail(), password); + getClient(token).perform(get("/api/core/items/" + publicItem1.getID()) + .param("projection", "full")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$._embedded.owningCollection._embedded.adminGroup", nullValue())); + + + getClient().perform(get("/api/core/items/" + publicItem1.getID()) + .param("projection", "full")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$._embedded.owningCollection._embedded.adminGroup").doesNotExist()); + + } + @Test public void findOneRelsTest() throws Exception { context.turnOffAuthorisationSystem(); diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkflowItemRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkflowItemRestRepositoryIT.java index e8c7922dc0..7a0fe01d4a 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkflowItemRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkflowItemRestRepositoryIT.java @@ -10,6 +10,7 @@ package org.dspace.app.rest; import static com.jayway.jsonpath.JsonPath.read; import static com.jayway.jsonpath.matchers.JsonPathMatchers.hasJsonPath; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.patch; @@ -1817,4 +1818,44 @@ public class WorkflowItemRestRepositoryIT extends AbstractControllerIntegrationT is("http://localhost/api/workflow/pooltask/search")) ))); } + + @Test + public void findOneFullProjectionTest() throws Exception { + context.turnOffAuthorisationSystem(); + + //** GIVEN ** + //1. A community-collection structure with one parent community with sub-community and two collections. + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Community child1 = CommunityBuilder.createSubCommunity(context, parentCommunity) + .withName("Sub Community") + .build(); + Collection col1 = CollectionBuilder.createCollection(context, child1).withName("Collection 1") + .withWorkflowGroup(1, admin).build(); + + //2. a workflow item + XmlWorkflowItem witem = WorkflowItemBuilder.createWorkflowItem(context, col1) + .withTitle("Workflow Item 1") + .withIssueDate("2017-10-17") + .withAuthor("Smith, Donald").withAuthor("Doe, John") + .withSubject("ExtraEntry") + .build(); + + context.restoreAuthSystemState(); + + String adminToken = getAuthToken(admin.getEmail(), password); + String epersonToken = getAuthToken(eperson.getEmail(), password); + + getClient(epersonToken).perform(get("/api/workflow/workflowitems/" + witem.getID()) + .param("projection", "full")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$._embedded.collection._embedded.adminGroup").doesNotExist()); + + getClient(adminToken).perform(get("/api/workflow/workflowitems/" + witem.getID()) + .param("projection", "full")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$._embedded.collection._embedded.adminGroup", nullValue())); + + } } diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java index d66cd463b9..9dd0fb2008 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java @@ -13,6 +13,7 @@ import static org.dspace.app.rest.matcher.MetadataMatcher.matchMetadata; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; import static org.springframework.data.rest.webmvc.RestMediaTypes.TEXT_URI_LIST_VALUE; import static org.springframework.http.MediaType.parseMediaType; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; @@ -4006,4 +4007,44 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration Matchers.is(WorkspaceItemMatcher.matchItemWithTitleAndDateIssuedAndSubject(witem, "Test title", "2019-04-25", "ExtraEntry")))); } + + @Test + public void findOneFullProjectionTest() throws Exception { + context.turnOffAuthorisationSystem(); + + //** GIVEN ** + //1. A community-collection structure with one parent community with sub-community and two collections. + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Community child1 = CommunityBuilder.createSubCommunity(context, parentCommunity) + .withName("Sub Community") + .build(); + Collection col1 = CollectionBuilder.createCollection(context, child1).withName("Collection 1").build(); + + //2. a workspace item + WorkspaceItem witem = WorkspaceItemBuilder.createWorkspaceItem(context, col1) + .withTitle("Workspace Item 1") + .withIssueDate("2017-10-17") + .withAuthor("Smith, Donald").withAuthor("Doe, John") + .withSubject("ExtraEntry") + .build(); + + context.restoreAuthSystemState(); + + String adminToken = getAuthToken(admin.getEmail(), password); + String epersonToken = getAuthToken(eperson.getEmail(), password); + + getClient(adminToken).perform(get("/api/submission/workspaceitems/" + witem.getID()) + .param("projection", "full")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$._embedded.collection._embedded.adminGroup", nullValue())); + + + getClient(epersonToken).perform(get("/api/submission/workspaceitems/" + witem.getID()) + .param("projection", "full")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$._embedded.collection._embedded.adminGroup").doesNotExist()); + + } } diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/matcher/CommunityMatcher.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/matcher/CommunityMatcher.java index 8b7e5669de..59f99794d8 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/matcher/CommunityMatcher.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/matcher/CommunityMatcher.java @@ -58,12 +58,23 @@ public class CommunityMatcher { ); } + public static Matcher matchCommunityEntryNonAdminEmbeds(String name, UUID uuid, String handle) { + return allOf( + matchProperties(name, uuid, handle), + hasJsonPath("$._embedded.collections", Matchers.not(Matchers.empty())), + hasJsonPath("$._embedded.logo", Matchers.not(Matchers.empty())), + matchLinks(uuid), + matchNonAdminEmbeds() + ); + } + public static Matcher matchCommunityEntryFullProjection(String name, UUID uuid, String handle) { return allOf( matchProperties(name, uuid, handle), hasJsonPath("$._embedded.collections", Matchers.not(Matchers.empty())), hasJsonPath("$._embedded.logo", Matchers.not(Matchers.empty())), - matchLinks(uuid) + matchLinks(uuid), + matchFullEmbeds() ); } @@ -82,7 +93,7 @@ public class CommunityMatcher { /** * Gets a matcher for all expected embeds when the full projection is requested. */ - public static Matcher matchFullEmbeds() { + public static Matcher matchNonAdminEmbeds() { return matchEmbeds( "collections[]", "logo", @@ -91,6 +102,19 @@ public class CommunityMatcher { ); } + /** + * Gets a matcher for all expected embeds when the full projection is requested. + */ + public static Matcher matchFullEmbeds() { + return matchEmbeds( + "collections[]", + "logo", + "parentCommunity", + "subcommunities[]", + "adminGroup" + ); + } + /** * Gets a matcher for all expected links. */ @@ -117,7 +141,7 @@ public class CommunityMatcher { ); } - public static String getFullEmbedsParameters() { + public static String getNonAdminEmbeds() { return "collections,logo,parentCommunity,subcommunities"; } From df252c40f8a9441409883dd0c7924fe2664ce0a2 Mon Sep 17 00:00:00 2001 From: Raf Ponsaerts Date: Tue, 18 Aug 2020 10:50:02 +0200 Subject: [PATCH 80/96] [Task 72546] bugfix on the author entries test --- .../app/rest/BrowsesResourceControllerIT.java | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/BrowsesResourceControllerIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/BrowsesResourceControllerIT.java index 31c1c11fca..0cf282b1ab 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/BrowsesResourceControllerIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/BrowsesResourceControllerIT.java @@ -1036,51 +1036,51 @@ public class BrowsesResourceControllerIT extends AbstractControllerIntegrationTe context.restoreAuthSystemState(); - getClient().perform(get("/api/discover/browses/title/items") + getClient().perform(get("/api/discover/browses/author/entries") .param("projection", "full")) - //** THEN ** - //The status has to be 200 OK .andExpect(status().isOk()) - //We expect the content type to be "application/hal+json;charset=UTF-8" .andExpect(content().contentType(contentType)) - //We expect 21 public items .andExpect(jsonPath("$.page.size", is(20))) .andExpect(jsonPath("$.page.totalElements", is(21))) .andExpect(jsonPath("$.page.totalPages", is(2))) .andExpect(jsonPath("$.page.number", is(0))) - // embedded items are already checked by other test, we focus on links here .andExpect( - jsonPath("$._links.next.href", Matchers.containsString("/api/discover/browses/title/items?"))) + jsonPath("$._links.next.href", Matchers.containsString("/api/discover/browses/author/entries"))) .andExpect( - jsonPath("$._links.last.href", Matchers.containsString("/api/discover/browses/title/items?"))) - .andExpect(jsonPath("$._links.self.href", Matchers.endsWith("/api/discover/browses/title/items"))) - // The full projection for anon shouldn't show the adminGroup in the response + jsonPath("$._links.last.href", Matchers.containsString("/api/discover/browses/author/entries"))) .andExpect( - jsonPath("$._embedded.items[0]._embedded.owningCollection._embedded.adminGroup").doesNotExist()); + jsonPath("$._links.self.href", Matchers.endsWith("/api/discover/browses/author/entries"))); String adminToken = getAuthToken(admin.getEmail(), password); - getClient(adminToken).perform(get("/api/discover/browses/title/items") + getClient(adminToken).perform(get("/api/discover/browses/author/entries") .param("projection", "full")) - //** THEN ** - //The status has to be 200 OK .andExpect(status().isOk()) - //We expect the content type to be "application/hal+json;charset=UTF-8" .andExpect(content().contentType(contentType)) - //We expect 21 public items .andExpect(jsonPath("$.page.size", is(20))) .andExpect(jsonPath("$.page.totalElements", is(21))) .andExpect(jsonPath("$.page.totalPages", is(2))) .andExpect(jsonPath("$.page.number", is(0))) - // embedded items are already checked by other test, we focus on links here .andExpect(jsonPath("$._links.next.href", - Matchers.containsString("/api/discover/browses/title/items?"))) + Matchers.containsString("/api/discover/browses/author/entries"))) .andExpect(jsonPath("$._links.last.href", - Matchers.containsString("/api/discover/browses/title/items?"))) + Matchers.containsString("/api/discover/browses/author/entries"))) .andExpect( - jsonPath("$._links.self.href", Matchers.endsWith("/api/discover/browses/title/items"))) - // The full projection for anon shouldn't show the adminGroup in the response - .andExpect(jsonPath("$._embedded.items[0]._embedded.owningCollection._embedded.adminGroup", - nullValue())); + jsonPath("$._links.self.href", + Matchers.endsWith("/api/discover/browses/author/entries"))); + + getClient().perform(get("/api/discover/browses/author/entries")) + .andExpect(status().isOk()) + .andExpect(content().contentType(contentType)) + .andExpect(jsonPath("$.page.size", is(20))) + .andExpect(jsonPath("$.page.totalElements", is(21))) + .andExpect(jsonPath("$.page.totalPages", is(2))) + .andExpect(jsonPath("$.page.number", is(0))) + .andExpect( + jsonPath("$._links.next.href", Matchers.containsString("/api/discover/browses/author/entries"))) + .andExpect( + jsonPath("$._links.last.href", Matchers.containsString("/api/discover/browses/author/entries"))) + .andExpect( + jsonPath("$._links.self.href", Matchers.endsWith("/api/discover/browses/author/entries"))); } From 5eaee53c6b8f598e6c1392dc12660b7505646fcf Mon Sep 17 00:00:00 2001 From: Samuel Date: Wed, 19 Aug 2020 15:16:59 +0200 Subject: [PATCH 81/96] taskid 72455 Cookie Preferences per account - feedback --- .../test/java/org/dspace/eperson/EPersonTest.java | 11 +++-------- dspace/config/registries/dspace-types.xml | 12 ++---------- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/dspace-api/src/test/java/org/dspace/eperson/EPersonTest.java b/dspace-api/src/test/java/org/dspace/eperson/EPersonTest.java index cd04fbb340..36d7d031ff 100644 --- a/dspace-api/src/test/java/org/dspace/eperson/EPersonTest.java +++ b/dspace-api/src/test/java/org/dspace/eperson/EPersonTest.java @@ -81,17 +81,12 @@ public class EPersonTest extends AbstractUnitTest { @Test public void testPreferences() throws Exception { - ePersonService.addMetadata(context, eperson, "dspace", "cookies", "functional", null, "true"); - ePersonService.addMetadata(context, eperson, "dspace", "cookies", "statistics", null, "false"); + ePersonService.addMetadata(context, eperson, "dspace", "agreements", "end-user", null, "test"); context.commit(); assertEquals( - "true", - ePersonService.getMetadataFirstValue(eperson, "dspace", "cookies", "functional", null) - ); - assertEquals( - "false", - ePersonService.getMetadataFirstValue(eperson, "dspace", "cookies", "statistics", null) + "test", + ePersonService.getMetadataFirstValue(eperson, "dspace", "agreements", "end-user", null) ); } diff --git a/dspace/config/registries/dspace-types.xml b/dspace/config/registries/dspace-types.xml index 3c3f460bb8..813998edd5 100644 --- a/dspace/config/registries/dspace-types.xml +++ b/dspace/config/registries/dspace-types.xml @@ -18,17 +18,9 @@ dspace - cookies - functional + agreements + end-user - - dspace - cookies - statistics - - - - From 899f04c5ed1651ce04c3a51e1442c9d9cd4bac81 Mon Sep 17 00:00:00 2001 From: jonas-atmire Date: Thu, 20 Aug 2020 13:06:02 +0200 Subject: [PATCH 82/96] Additional metadatafield related to the cookie preferences --- .../src/test/java/org/dspace/eperson/EPersonTest.java | 5 +++++ dspace/config/registries/dspace-types.xml | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/dspace-api/src/test/java/org/dspace/eperson/EPersonTest.java b/dspace-api/src/test/java/org/dspace/eperson/EPersonTest.java index 36d7d031ff..a47974a5e4 100644 --- a/dspace-api/src/test/java/org/dspace/eperson/EPersonTest.java +++ b/dspace-api/src/test/java/org/dspace/eperson/EPersonTest.java @@ -82,12 +82,17 @@ public class EPersonTest extends AbstractUnitTest { public void testPreferences() throws Exception { ePersonService.addMetadata(context, eperson, "dspace", "agreements", "end-user", null, "test"); + ePersonService.addMetadata(context, eperson, "dspace", "agreements", "cookies", null, "Dspace cookies agreement metadata"); context.commit(); assertEquals( "test", ePersonService.getMetadataFirstValue(eperson, "dspace", "agreements", "end-user", null) ); + assertEquals( + "Dspace cookies agreement metadata", + ePersonService.getMetadataFirstValue(eperson, "dspace", "agreements", "cookies", null) + ); } /** diff --git a/dspace/config/registries/dspace-types.xml b/dspace/config/registries/dspace-types.xml index 813998edd5..75c4ff84cd 100644 --- a/dspace/config/registries/dspace-types.xml +++ b/dspace/config/registries/dspace-types.xml @@ -23,4 +23,11 @@ + + dspace + agreements + cookies + + + From 1773b40acb9fe65488abc59e8c101c66767650c8 Mon Sep 17 00:00:00 2001 From: jonas-atmire Date: Thu, 20 Aug 2020 14:53:31 +0200 Subject: [PATCH 83/96] Checkstyle fix --- dspace-api/src/test/java/org/dspace/eperson/EPersonTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dspace-api/src/test/java/org/dspace/eperson/EPersonTest.java b/dspace-api/src/test/java/org/dspace/eperson/EPersonTest.java index a47974a5e4..99d673d4fa 100644 --- a/dspace-api/src/test/java/org/dspace/eperson/EPersonTest.java +++ b/dspace-api/src/test/java/org/dspace/eperson/EPersonTest.java @@ -82,7 +82,8 @@ public class EPersonTest extends AbstractUnitTest { public void testPreferences() throws Exception { ePersonService.addMetadata(context, eperson, "dspace", "agreements", "end-user", null, "test"); - ePersonService.addMetadata(context, eperson, "dspace", "agreements", "cookies", null, "Dspace cookies agreement metadata"); + ePersonService.addMetadata(context, eperson, "dspace", "agreements", "cookies", null, + "Dspace cookies agreement metadata"); context.commit(); assertEquals( From 2f26cf7def6c36147d80050a9967ae1e83c96c8c Mon Sep 17 00:00:00 2001 From: Marie Verdonck Date: Fri, 21 Aug 2020 16:07:07 +0200 Subject: [PATCH 84/96] 71752: Refactored metadatafield indexing via event > consumer --- .../content/MetadataFieldServiceImpl.java | 33 ++++++++++--------- .../dspace/discovery/IndexEventConsumer.java | 24 ++++++++++++-- .../MetadataFieldRestRepository.java | 6 ++-- dspace/config/dspace.cfg | 2 +- 4 files changed, 45 insertions(+), 20 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/content/MetadataFieldServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/MetadataFieldServiceImpl.java index bfd9942736..569b5840c6 100644 --- a/dspace-api/src/main/java/org/dspace/content/MetadataFieldServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/MetadataFieldServiceImpl.java @@ -9,6 +9,8 @@ package org.dspace.content; import java.io.IOException; import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import org.apache.commons.collections4.CollectionUtils; @@ -20,11 +22,12 @@ import org.dspace.content.dao.MetadataFieldDAO; import org.dspace.content.service.MetadataFieldService; import org.dspace.content.service.MetadataSchemaService; import org.dspace.content.service.MetadataValueService; +import org.dspace.content.service.SiteService; +import org.dspace.core.Constants; import org.dspace.core.Context; import org.dspace.core.LogManager; -import org.dspace.discovery.IndexingService; import org.dspace.discovery.indexobject.IndexableMetadataField; -import org.dspace.services.factory.DSpaceServicesFactory; +import org.dspace.event.Event; import org.springframework.beans.factory.annotation.Autowired; /** @@ -49,7 +52,8 @@ public class MetadataFieldServiceImpl implements MetadataFieldService { protected MetadataValueService metadataValueService; @Autowired(required = true) protected MetadataSchemaService metadataSchemaService; - private IndexingService indexer; + @Autowired + protected SiteService siteService; protected MetadataFieldServiceImpl() { @@ -82,7 +86,7 @@ public class MetadataFieldServiceImpl implements MetadataFieldService { log.info(LogManager.getHeader(context, "create_metadata_field", "metadata_field_id=" + metadataField.getID())); // Update the index of type metadatafield - this.updateMetadataFieldIndex(context); + this.triggerEventToUpdateIndex(context, metadataField.getID()); return metadataField; } @@ -156,7 +160,7 @@ public class MetadataFieldServiceImpl implements MetadataFieldService { .getElement() + "qualifier=" + metadataField.getQualifier())); // Update the index of type metadatafield - this.updateMetadataFieldIndex(context); + this.triggerEventToUpdateIndex(context, metadataField.getID()); } @Override @@ -186,21 +190,20 @@ public class MetadataFieldServiceImpl implements MetadataFieldService { log.info(LogManager.getHeader(context, "delete_metadata_field", "metadata_field_id=" + metadataField.getID())); // Update the index of type metadatafield - this.updateMetadataFieldIndex(context); + this.triggerEventToUpdateIndex(context, metadataField.getID()); } /** - * Updates the index of type metadatafield. Indexed metadatafields are used by the /search/byFieldName endpoint, - * see MetadataFieldRestRepository + * Calls a MODIFY SITE event with the identifier of the changed mdField, so it can be indexed in + * {@link org.dspace.discovery.IndexEventConsumer}, with type of {@link org.dspace.discovery.IndexableObject} in + * {@link Event}.detail and the identifiers of the changed mdFields in {@link Event}.identifiers + * * @param context DSpace context + * @param mdFieldId ID of the metadata field that needs to be (re)indexed */ - private void updateMetadataFieldIndex(Context context) { - if (this.indexer == null) { - this.indexer = DSpaceServicesFactory.getInstance().getServiceManager() - .getServiceByName(IndexingService.class.getName(), - IndexingService.class); - } - this.indexer.updateIndex(context, true, IndexableMetadataField.TYPE); + private void triggerEventToUpdateIndex(Context context, int mdFieldId) { + context.addEvent(new Event(Event.MODIFY, Constants.SITE, null, IndexableMetadataField.TYPE, new ArrayList<>( + Arrays.asList(Integer.toString(mdFieldId))))); } /** diff --git a/dspace-api/src/main/java/org/dspace/discovery/IndexEventConsumer.java b/dspace-api/src/main/java/org/dspace/discovery/IndexEventConsumer.java index 43ea9eefb2..195c9cd6fc 100644 --- a/dspace-api/src/main/java/org/dspace/discovery/IndexEventConsumer.java +++ b/dspace-api/src/main/java/org/dspace/discovery/IndexEventConsumer.java @@ -8,6 +8,7 @@ package org.dspace.discovery; import java.util.HashSet; +import java.util.Optional; import java.util.Set; import org.apache.logging.log4j.Logger; @@ -15,6 +16,7 @@ import org.dspace.content.Bundle; import org.dspace.content.DSpaceObject; import org.dspace.core.Constants; import org.dspace.core.Context; +import org.dspace.discovery.indexobject.factory.IndexFactory; import org.dspace.discovery.indexobject.factory.IndexObjectFactoryFactory; import org.dspace.event.Consumer; import org.dspace.event.Event; @@ -67,7 +69,7 @@ public class IndexEventConsumer implements Consumer { int st = event.getSubjectType(); if (!(st == Constants.ITEM || st == Constants.BUNDLE - || st == Constants.COLLECTION || st == Constants.COMMUNITY)) { + || st == Constants.COLLECTION || st == Constants.COMMUNITY || st == Constants.SITE)) { log .warn("IndexConsumer should not have been given this kind of Subject in an event, skipping: " + event.toString()); @@ -104,10 +106,28 @@ public class IndexEventConsumer implements Consumer { case Event.MODIFY: case Event.MODIFY_METADATA: if (subject == null) { - log.warn(event.getEventTypeAsString() + " event, could not get object for " + if (st == Constants.SITE) { + // Update the indexable objects of type in event.detail of objects with ids in event.identifiers + for (String id : event.getIdentifiers()) { + IndexFactory indexableObjectService = IndexObjectFactoryFactory.getInstance(). + getIndexFactoryByType(event.getDetail()); + Optional indexableObject = Optional.empty(); + indexableObject = indexableObjectService.findIndexableObject(ctx, id); + if (indexableObject.isPresent()) { + log.debug("consume() adding event to update queue: " + event.toString()); + objectsToUpdate + .addAll(indexObjectServiceFactory + .getIndexableObjects(ctx, indexableObject.get().getIndexedObject())); + } else { + log.warn("Cannot resolve " + id); + } + } + } else { + log.warn(event.getEventTypeAsString() + " event, could not get object for " + event.getSubjectTypeAsString() + " id=" + event.getSubjectID() + ", perhaps it has been deleted."); + } } else { log.debug("consume() adding event to update queue: " + event.toString()); objectsToUpdate.addAll(indexObjectServiceFactory.getIndexableObjects(ctx, subject)); diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataFieldRestRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataFieldRestRepository.java index 31ea246efe..b0a5f526f0 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataFieldRestRepository.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataFieldRestRepository.java @@ -140,7 +140,8 @@ public class MetadataFieldRestRepository extends DSpaceRestRepository Date: Fri, 21 Aug 2020 10:34:29 -0500 Subject: [PATCH 85/96] Add CLI examples and how to change configs --- dspace/src/main/docker-compose/README.md | 25 ++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/dspace/src/main/docker-compose/README.md b/dspace/src/main/docker-compose/README.md index 9c92f627b6..372a03a6c5 100644 --- a/dspace/src/main/docker-compose/README.md +++ b/dspace/src/main/docker-compose/README.md @@ -84,3 +84,28 @@ Download an assetstore from a tar file on the internet. ``` docker-compose -p d7 -f docker-compose-cli.yml -f dspace/src/main/docker-compose/cli.assetstore.yml run dspace-cli ``` + +## Modify DSpace Configuration in Docker +While your Docker containers are running, you may directly modify the `local.cfg` in this directory which will change the DSpace configuration for the running Docker container. (Keep in mind, this works because our `docker-compose.yml` mounts this `[src]/dspace/src/main/docker-compose/local.cfg` from the host into the running Docker instance.) + +Many DSpace configuration settings will reload automatically (after a few seconds). However, configurations which are cached by DSpace (or by Spring Boot) may require you to quickly reboot the Docker containers by running `docker-compose -p d7 down` followed by `docker-compose -p d7 up -d`. + +## Running DSpace CLI scripts in Docker +While the Docker containers are running, you can use the DSpace CLI image to run any DSpace commandline script (i.e. any command that normally can be run by `[dspace]/bin/dspace`). The general format is: + +``` +docker-compose -p d7 -f docker-compose-cli.yml run --rm dspace-cli [command] [parameters] +``` + +So, for example, to reindex all content in Discovery, normally you'd run `./dspace index-discovery -b` from commandline. Using our DSpace CLI image, that command becomes: + +``` +docker-compose -p d7 -f docker-compose-cli.yml run --rm dspace-cli index-discovery -b +``` + +Similarly, you can see the value of any DSpace configuration (in local.cfg or dspace.cfg) by running: + +``` +# Output the value of `dspace.ui.url` from running Docker instance +docker-compose -p d7 -f docker-compose-cli.yml run --rm dspace-cli dsprop -p dspace.ui.url +``` From 4b86f331d43041d3991124534c4c546068d75546 Mon Sep 17 00:00:00 2001 From: Marie Verdonck Date: Wed, 26 Aug 2020 13:52:59 +0200 Subject: [PATCH 86/96] 72505: Url sitemaps fix --- .../src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java b/dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java index b16630e853..e2743951e7 100644 --- a/dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java +++ b/dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java @@ -181,9 +181,8 @@ public class GenerateSitemaps { * if IO error occurs. */ public static void generateSitemaps(boolean makeHTMLMap, boolean makeSitemapOrg) throws SQLException, IOException { - String sitemapsEndpoint = configurationService.getProperty("sitemap.path", "sitemaps"); String uiURLStem = configurationService.getProperty("dspace.ui.url"); - String sitemapStem = uiURLStem + "/" + sitemapsEndpoint + "/sitemap"; + String sitemapStem = uiURLStem + "/sitemap"; File outputDir = new File(configurationService.getProperty("sitemap.dir")); if (!outputDir.exists() && !outputDir.mkdir()) { From fdc0cda30b065141ec6722d90bae4763bfe98624 Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo <37987333+pasqualecvl@users.noreply.github.com> Date: Wed, 26 Aug 2020 14:27:56 +0200 Subject: [PATCH 87/96] Update pom.xml Add comment --- dspace-api/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspace-api/pom.xml b/dspace-api/pom.xml index 68c822423c..41ad956d82 100644 --- a/dspace-api/pom.xml +++ b/dspace-api/pom.xml @@ -325,7 +325,7 @@ apache-jena-libs pom - + org.glassfish.jersey.inject From 4d3fd877225e9fc6cedd3f886bde4b9d067c8d07 Mon Sep 17 00:00:00 2001 From: Marie Verdonck Date: Wed, 26 Aug 2020 15:05:18 +0200 Subject: [PATCH 88/96] 72505: Url sitemaps change - test fix --- .../java/org/dspace/app/rest/SitemapRestControllerIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/SitemapRestControllerIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/SitemapRestControllerIT.java index d51992e9be..fd84aa023b 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/SitemapRestControllerIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/SitemapRestControllerIT.java @@ -112,7 +112,7 @@ public class SitemapRestControllerIT extends AbstractControllerIntegrationTest { String response = result.getResponse().getContentAsString(); // contains a link to /sitemaps/sitemap0.html - assertTrue(response.contains("/" + SITEMAPS_ENDPOINT + "/sitemap0.html")); + assertTrue(response.contains("/sitemap0.html")); } @Test @@ -145,7 +145,7 @@ public class SitemapRestControllerIT extends AbstractControllerIntegrationTest { String response = result.getResponse().getContentAsString(); // contains a link to /sitemaps/sitemap0.html - assertTrue(response.contains("/" + SITEMAPS_ENDPOINT + "/sitemap0.xml")); + assertTrue(response.contains("/sitemap0.xml")); } @Test From 44fc6922c36bef94395af56e9677e6584f182421 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Wed, 26 Aug 2020 12:16:40 -0500 Subject: [PATCH 89/96] Add action to auto-label PRs with merge conflicts --- .github/workflows/label_merge_conflicts.yml | 23 +++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/label_merge_conflicts.yml diff --git a/.github/workflows/label_merge_conflicts.yml b/.github/workflows/label_merge_conflicts.yml new file mode 100644 index 0000000000..9a12213b48 --- /dev/null +++ b/.github/workflows/label_merge_conflicts.yml @@ -0,0 +1,23 @@ +# This workflow checks open PRs for merge conflicts and labels them when conflicts are found +name: Check for merge conflicts + +# Run whenever the "main" branch is updated +# NOTE: This means merge conflicts are only checked for when a PR is merged to main. +on: + push: + branches: + - main + +jobs: + triage: + runs-on: ubuntu-latest + steps: + # See: https://github.com/mschilde/auto-label-merge-conflicts/ + - name: Auto-label PRs with merge conflicts + uses: mschilde/auto-label-merge-conflicts@v2.0 + # Add "merge conflict" label if a merge conflict is detected. Remove it when resolved. + # Note, the authentication token is created automatically + # See: https://docs.github.com/en/actions/configuring-and-managing-workflows/authenticating-with-the-github_token + with: + CONFLICT_LABEL_NAME: 'merge conflict' + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 5718d78ab59b325778a3440cdc645d80c9424243 Mon Sep 17 00:00:00 2001 From: Yana De Pauw Date: Thu, 27 Aug 2020 15:09:44 +0200 Subject: [PATCH 90/96] 72751: Remove log.info line from builder --- .../java/org/dspace/builder/AbstractDSpaceObjectBuilder.java | 1 - 1 file changed, 1 deletion(-) diff --git a/dspace-api/src/test/java/org/dspace/builder/AbstractDSpaceObjectBuilder.java b/dspace-api/src/test/java/org/dspace/builder/AbstractDSpaceObjectBuilder.java index 075a533273..69cfd0e136 100644 --- a/dspace-api/src/test/java/org/dspace/builder/AbstractDSpaceObjectBuilder.java +++ b/dspace-api/src/test/java/org/dspace/builder/AbstractDSpaceObjectBuilder.java @@ -164,7 +164,6 @@ public abstract class AbstractDSpaceObjectBuilder eperson, startDate, Constants.ADMIN, "Integration Test", dso); if (rp != null) { - log.info("Updating resource policy with REMOVE for eperson: " + eperson.getEmail()); resourcePolicyService.update(context, rp); } } catch (Exception e) { From 7422dafef5d511281b808a333785e5598aec5bc4 Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Thu, 27 Aug 2020 16:22:29 +0200 Subject: [PATCH 91/96] Change javadoc on ArXivIdMetadataContributor.parseValue --- .../contributor/ArXivIdMetadataContributor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/contributor/ArXivIdMetadataContributor.java b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/contributor/ArXivIdMetadataContributor.java index 077fe85675..aaea53ecfc 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/contributor/ArXivIdMetadataContributor.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/contributor/ArXivIdMetadataContributor.java @@ -39,7 +39,8 @@ public class ArXivIdMetadataContributor extends SimpleXpathMetadatumContributor } /** - * ArXiv doesn't return the item id. We have to get this from the path parameter + * ArXiv returns a full URL as in the value, e.g. http://arxiv.org/abs/1911.11405v1. + * This method parses out the identifier from the end of the URL, e.g. 1911.11405v1. * * @param dtos Metadata which contains the items uri * @return the items ids From 8adc7e5c318bd09dcf38133319da61d67b261328 Mon Sep 17 00:00:00 2001 From: Samuel Date: Fri, 28 Aug 2020 15:26:22 +0200 Subject: [PATCH 92/96] taskid 72455 Cookie Preferences per account - add scope notes and more realistic test value --- .../java/org/dspace/eperson/EPersonTest.java | 23 +++++++++++++------ dspace/config/registries/dspace-types.xml | 4 ++-- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/dspace-api/src/test/java/org/dspace/eperson/EPersonTest.java b/dspace-api/src/test/java/org/dspace/eperson/EPersonTest.java index 99d673d4fa..7603500e8e 100644 --- a/dspace-api/src/test/java/org/dspace/eperson/EPersonTest.java +++ b/dspace-api/src/test/java/org/dspace/eperson/EPersonTest.java @@ -81,18 +81,27 @@ public class EPersonTest extends AbstractUnitTest { @Test public void testPreferences() throws Exception { - ePersonService.addMetadata(context, eperson, "dspace", "agreements", "end-user", null, "test"); - ePersonService.addMetadata(context, eperson, "dspace", "agreements", "cookies", null, - "Dspace cookies agreement metadata"); + String cookies = + "{" + + "\"token_item\":true," + + "\"impersonation\":true," + + "\"redirect\":true," + + "\"language\":true," + + "\"klaro\":true," + + "\"google-analytics\":false" + + "}"; + + ePersonService.addMetadata(context, eperson, "dspace", "agreements", "cookies", null, cookies); + ePersonService.addMetadata(context, eperson, "dspace", "agreements", "end-user", null, "true"); context.commit(); assertEquals( - "test", - ePersonService.getMetadataFirstValue(eperson, "dspace", "agreements", "end-user", null) + cookies, + ePersonService.getMetadataFirstValue(eperson, "dspace", "agreements", "cookies", null) ); assertEquals( - "Dspace cookies agreement metadata", - ePersonService.getMetadataFirstValue(eperson, "dspace", "agreements", "cookies", null) + "true", + ePersonService.getMetadataFirstValue(eperson, "dspace", "agreements", "end-user", null) ); } diff --git a/dspace/config/registries/dspace-types.xml b/dspace/config/registries/dspace-types.xml index 75c4ff84cd..f88def2453 100644 --- a/dspace/config/registries/dspace-types.xml +++ b/dspace/config/registries/dspace-types.xml @@ -20,14 +20,14 @@ dspace agreements end-user - + Stores whether the End User Agreement has been accepted by an EPerson. Valid values; true, false dspace agreements cookies - + Stores the cookie preferences of an EPerson, as selected in last session. Value will be an array of cookieName/boolean pairs, specifying which cookies are allowed or not allowed. From 0746caf00dd1f72c70fa4cc28ef4be154bb54215 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Tue, 1 Sep 2020 09:09:20 -0500 Subject: [PATCH 93/96] Update action to ignore errors --- .github/workflows/label_merge_conflicts.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/label_merge_conflicts.yml b/.github/workflows/label_merge_conflicts.yml index 9a12213b48..dcbab18f1b 100644 --- a/.github/workflows/label_merge_conflicts.yml +++ b/.github/workflows/label_merge_conflicts.yml @@ -21,3 +21,5 @@ jobs: with: CONFLICT_LABEL_NAME: 'merge conflict' GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Ignore errors + continue-on-error: true From daeb55fa0c52eb61fe73cecc44d5d8f20d334d97 Mon Sep 17 00:00:00 2001 From: "Mark H. Wood" Date: Wed, 2 Sep 2020 16:19:37 -0400 Subject: [PATCH 94/96] Use getters instead of direct reference that may throw NPE. #2951 --- .../java/org/dspace/identifier/DOIIdentifierProvider.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/identifier/DOIIdentifierProvider.java b/dspace-api/src/main/java/org/dspace/identifier/DOIIdentifierProvider.java index 46bc317d13..9db4402007 100644 --- a/dspace-api/src/main/java/org/dspace/identifier/DOIIdentifierProvider.java +++ b/dspace-api/src/main/java/org/dspace/identifier/DOIIdentifierProvider.java @@ -761,9 +761,9 @@ public class DOIIdentifierProvider Item item = (Item) dso; List metadata = itemService.getMetadata(item, MD_SCHEMA, DOI_ELEMENT, DOI_QUALIFIER, null); + String leftPart = DOI.RESOLVER + SLASH + getPrefix() + SLASH + getNamespaceSeparator(); for (MetadataValue id : metadata) { - if (id.getValue().startsWith( - DOI.RESOLVER + String.valueOf(SLASH) + PREFIX + String.valueOf(SLASH) + NAMESPACE_SEPARATOR)) { + if (id.getValue().startsWith(leftPart)) { return doiService.DOIFromExternalFormat(id.getValue()); } } From 1b4988d9d6df0b0c7dc87ed56e142be9d7bb736e Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Thu, 3 Sep 2020 17:25:02 +0200 Subject: [PATCH 95/96] remove wrong comment --- .../metadatamapping/contributor/ArXivIdMetadataContributor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/contributor/ArXivIdMetadataContributor.java b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/contributor/ArXivIdMetadataContributor.java index aaea53ecfc..ed5ac5960b 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/contributor/ArXivIdMetadataContributor.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/arxiv/metadatamapping/contributor/ArXivIdMetadataContributor.java @@ -43,7 +43,6 @@ public class ArXivIdMetadataContributor extends SimpleXpathMetadatumContributor * This method parses out the identifier from the end of the URL, e.g. 1911.11405v1. * * @param dtos Metadata which contains the items uri - * @return the items ids */ private void parseValue(Collection dtos) { if (dtos != null) { From 280a4dc238392541f487017306ab9ad84798d315 Mon Sep 17 00:00:00 2001 From: Tim Donohue Date: Tue, 8 Sep 2020 11:59:56 -0500 Subject: [PATCH 96/96] Ensure prior data is cleaned. Fix ordering of params in assert --- dspace-api/src/test/java/org/dspace/curate/CuratorTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dspace-api/src/test/java/org/dspace/curate/CuratorTest.java b/dspace-api/src/test/java/org/dspace/curate/CuratorTest.java index 4890fef26f..0abb3b48ac 100644 --- a/dspace-api/src/test/java/org/dspace/curate/CuratorTest.java +++ b/dspace-api/src/test/java/org/dspace/curate/CuratorTest.java @@ -54,7 +54,7 @@ public class CuratorTest extends AbstractUnitTest { */ @Test public void testCurate_DSpaceObject() throws Exception { - System.out.println("curate"); + CoreServiceFactory.getInstance().getPluginService().clearNamedPluginClasses(); final String TASK_NAME = "dummyTask"; @@ -114,6 +114,6 @@ public class CuratorTest extends AbstractUnitTest { curator.curate(context, item); assertEquals(Curator.CURATE_SUCCESS, curator.getStatus(TASK_NAME)); - assertEquals(reporterOutput.toString(), "No operation performed on testHandle"); + assertEquals("No operation performed on testHandle", reporterOutput.toString()); } }