coar-notify-7 incoming ldn message block configs, new item-filters map, hide coarnotify section on submission, IT fixes

dspace-api/src/main/java/org/dspace/app/ldn/dao/impl/LDNMessageDaoImpl.java

@@ -128,8 +128,6 @@ public class LDNMessageDaoImpl extends AbstractHibernateDAO<LDNMessageEntity> im
             criteriaBuilder.equal(root.get(LDNMessageEntity_.queueStatus), LDNMessageEntity.QUEUE_STATUS_PROCESSED));
         andPredicates.add(
             criteriaBuilder.equal(root.get(LDNMessageEntity_.object), item));
-        andPredicates.add(
-            criteriaBuilder.isNull(root.get(LDNMessageEntity_.origin)));
         if (activities != null && activities.length > 0) {
             activityPredicate = root.get(LDNMessageEntity_.activityStreamType).in(activities);
             andPredicates.add(activityPredicate);

dspace-api/src/main/java/org/dspace/app/ldn/service/LDNMessageService.java

@@ -128,6 +128,7 @@ public interface LDNMessageService {
      * @throws SQLException If something goes wrong in the database
      */
     public NotifyRequestStatus findRequestsByItem(Context context, Item item) throws SQLException;
+
     /**
      * delete the provided ldn message
      *
@@ -136,4 +137,12 @@ public interface LDNMessageService {
      * @throws SQLException if something goes wrong
      */
     public void delete(Context context, LDNMessageEntity ldnMessage) throws SQLException;
+
+    /**
+     * check if IP number is included in the configured ip-range on the Notify Service
+     * 
+     * @param origin the Notify Service entity
+     * @param sourceIp the ip to evaluate
+     */
+    public boolean isValidIp(NotifyServiceEntity origin, String sourceIp);
 }

dspace-api/src/main/java/org/dspace/app/ldn/service/impl/LDNMessageServiceImpl.java

@@ -97,7 +97,6 @@ public class LDNMessageServiceImpl implements LDNMessageService {
             ldnMessage.setContext(findDspaceObjectByUrl(context, notification.getContext().getId()));
         }
         ldnMessage.setOrigin(findNotifyService(context, notification.getOrigin()));
-        ldnMessage.setTarget(findNotifyService(context, notification.getTarget()));
         ldnMessage.setInReplyTo(find(context, notification.getInReplyTo()));
         ObjectMapper mapper = new ObjectMapper();
         String message = null;
@@ -123,36 +122,30 @@ public class LDNMessageServiceImpl implements LDNMessageService {
             ldnMessage.setCoarNotifyType(notificationTypeArrayList.get(1));
         }
         ldnMessage.setQueueStatus(LDNMessageEntity.QUEUE_STATUS_QUEUED);
+        ldnMessage.setSourceIp(sourceIp);
         if (ldnMessage.getOrigin() == null && !"Offer".equalsIgnoreCase(ldnMessage.getActivityStreamType())) {
             ldnMessage.setQueueStatus(LDNMessageEntity.QUEUE_STATUS_UNTRUSTED);
         } else {
-            if (!isValidIp(ldnMessage)) {
+
+            boolean ipCheckRangeEnabled = configurationService.getBooleanProperty("ldn.ip-range.enabled", true);
+            if (ipCheckRangeEnabled && !isValidIp(ldnMessage.getOrigin(), sourceIp)) {
                 ldnMessage.setQueueStatus(LDNMessageEntity.QUEUE_STATUS_UNTRUSTED_IP);
             }
         }
         ldnMessage.setQueueTimeout(new Date());
-        ldnMessage.setSourceIp(sourceIp);
 
         update(context, ldnMessage);
         return ldnMessage;
     }
 
-    private boolean isValidIp(LDNMessageEntity message) {
+    @Override
+    public boolean isValidIp(NotifyServiceEntity origin, String sourceIp) {
 
-        boolean enabled = configurationService.getBooleanProperty("coar-notify.ip-range.enabled", true);
+        String lowerIp = origin.getLowerIp();
-
+        String upperIp = origin.getUpperIp();
-        if (!enabled) {
-            return true;
-        }
-
-        NotifyServiceEntity notifyService =
-            message.getOrigin() == null ? message.getTarget() : message.getOrigin();
-
-        String lowerIp = notifyService.getLowerIp();
-        String upperIp = notifyService.getUpperIp();
 
         try {
-            InetAddress ip = InetAddress.getByName(message.getSourceIp());
+            InetAddress ip = InetAddress.getByName(sourceIp);
             InetAddress lowerBoundAddress = InetAddress.getByName(lowerIp);
             InetAddress upperBoundAddress = InetAddress.getByName(upperIp);
 
@@ -333,8 +326,8 @@ public class LDNMessageServiceImpl implements LDNMessageService {
         if (msgs != null && !msgs.isEmpty()) {
             for (LDNMessageEntity msg : msgs) {
                 RequestStatus offer = new RequestStatus();
-                offer.setServiceName(msg.getTarget() == null ? "Unknown Service" : msg.getTarget().getName());
+                offer.setServiceName(msg.getOrigin() == null ? "Unknown Service" : msg.getOrigin().getName());
-                offer.setServiceUrl(msg.getTarget() == null ? "" : msg.getTarget().getUrl());
+                offer.setServiceUrl(msg.getOrigin() == null ? "" : msg.getOrigin().getUrl());
                 offer.setOfferType(LDNUtils.getNotifyType(msg.getCoarNotifyType()));
                 List<LDNMessageEntity> acks = ldnMessageDao.findAllRelatedMessagesByItem(
                     context, msg, item, "Accept", "TentativeReject", "TentativeAccept", "Announce");

dspace-api/src/main/java/org/dspace/content/ItemFilterServiceImpl.java

@@ -8,6 +8,7 @@
 package org.dspace.content;
 
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 import org.dspace.app.ldn.ItemFilter;
@@ -37,12 +38,21 @@ public class ItemFilterServiceImpl implements ItemFilterService {
 
     @Override
     public List<ItemFilter> findAll() {
-        return serviceManager.getServicesWithNamesByType(LogicalStatement.class)
+        Map<String, LogicalStatement> ldnFilters =
-                             .keySet()
+            serviceManager.getServiceByName("ldnItemFilters", Map.class);
-                             .stream()
+        return ldnFilters.keySet()
-                             .sorted()
+            .stream()
-                             .map(ItemFilter::new)
+            .sorted()
-                             .collect(Collectors.toList());
+            .map(ItemFilter::new)
+            .collect(Collectors.toList());
+    }
+
+    public ServiceManager getServiceManager() {
+        return serviceManager;
+    }
+
+    public void setServiceManager(ServiceManager serviceManager) {
+        this.serviceManager = serviceManager;
     }
 
 }

dspace-api/src/test/data/dspaceFolder/config/local.cfg

@@ -174,3 +174,13 @@ authority.controlled.dspace.object.owner = true
 # Configuration required for thorough testing of browse links
 webui.browse.link.1 = author:dc.contributor.*
 webui.browse.link.2 = subject:dc.subject.*
+
+
+###########################################
+#  LDN CONFIGURATIONS                     #
+###########################################
+ldn.enabled = true
+qaevents.enabled = true
+ldn.ip-range.enabled = true
+ldn.notify.inbox.block-untrusted = true
+ldn.notify.inbox.block-untrusted-ip = true

dspace-api/src/test/data/dspaceFolder/config/spring/api/item-filters.xml

@@ -367,4 +367,12 @@
         </property>
     </bean>
 
+    <util:map id="ldnItemFilters" key-type="java.lang.String"
+              value-type="org.dspace.content.logic.LogicalStatement">
+       <entry key="always_true_filter" value-ref="always_true_filter"/>
+       <entry key="in-outfit-collection_condition" value-ref="in-outfit-collection_condition"/>
+       <entry key="demo_filter" value-ref="demo_filter"/>
+       <entry key="doi-filter" value-ref="doi-filter"/>
+       <entry key="type_filter" value-ref="type_filter"/>
+	</util:map>
 </beans>

dspace-server-webapp/src/main/java/org/dspace/app/rest/LDNInboxController.java

@@ -7,6 +7,7 @@
  */
 package org.dspace.app.rest;
 
+import java.sql.SQLException;
 import java.util.regex.Pattern;
 import javax.servlet.http.HttpServletRequest;
 
@@ -14,10 +15,13 @@ import org.apache.commons.validator.routines.UrlValidator;
 import org.apache.logging.log4j.Logger;
 import org.dspace.app.ldn.LDNMessageEntity;
 import org.dspace.app.ldn.LDNRouter;
+import org.dspace.app.ldn.NotifyServiceEntity;
 import org.dspace.app.ldn.model.Notification;
 import org.dspace.app.ldn.service.LDNMessageService;
+import org.dspace.app.rest.exception.DSpaceBadRequestException;
 import org.dspace.app.rest.exception.InvalidLDNMessageException;
 import org.dspace.core.Context;
+import org.dspace.services.ConfigurationService;
 import org.dspace.web.ContextUtil;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
@@ -44,6 +48,9 @@ public class LDNInboxController {
     @Autowired
     private LDNMessageService ldnMessageService;
 
+    @Autowired
+    private ConfigurationService configurationService;
+
     /**
      * LDN DSpace inbox.
      *
@@ -56,8 +63,7 @@ public class LDNInboxController {
         throws Exception {
 
         Context context = ContextUtil.obtainCurrentRequestContext();
-        validate(notification);
+        validate(context, notification, request.getRemoteAddr());
-        log.info("stored notification {} {}", notification.getId(), notification.getType());
 
         LDNMessageEntity ldnMsgEntity = ldnMessageService.create(context, notification, request.getRemoteAddr());
         log.info("stored ldn message {}", ldnMsgEntity);
@@ -91,7 +97,7 @@ public class LDNInboxController {
                 .body(e.getMessage());
     }
 
-    private void validate(Notification notification) {
+    private void validate(Context context, Notification notification, String sourceIp) {
         String id = notification.getId();
         Pattern URNRegex =
             Pattern.compile("^urn:uuid:[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$");
@@ -103,6 +109,37 @@ public class LDNInboxController {
         if (notification.getOrigin() == null || notification.getTarget() == null || notification.getObject() == null) {
             throw new InvalidLDNMessageException("Origin or Target or Object is missing");
         }
-    }
 
+        if (configurationService.getBooleanProperty("ldn.notify.inbox.block-untrusted", true)) {
+            try {
+                NotifyServiceEntity originNotifyService =
+                    ldnMessageService.findNotifyService(context, notification.getOrigin());
+                if (originNotifyService == null) {
+                    throw new DSpaceBadRequestException("Notify Service [" + notification.getOrigin()
+                        + "] unknown. LDN message can not be received.");
+                }
+            } catch (SQLException sqle) {
+                throw new DSpaceBadRequestException("Notify Service [" + notification.getOrigin()
+                + "] unknown. LDN message can not be received.");
+            }
+        }
+        if (configurationService.getBooleanProperty("ldn.notify.inbox.block-untrusted-ip", true)) {
+            try {
+                NotifyServiceEntity originNotifyService =
+                    ldnMessageService.findNotifyService(context, notification.getOrigin());
+                if (originNotifyService == null) {
+                    throw new DSpaceBadRequestException("Notify Service [" + notification.getOrigin()
+                        + "] unknown. LDN message can not be received.");
+                }
+                boolean isValidIp = ldnMessageService.isValidIp(originNotifyService, sourceIp);
+                if (!isValidIp) {
+                    throw new DSpaceBadRequestException("Source IP for Incoming LDN Message [" + notification.getId()
+                        + "] out of its Notify Service IP Range. LDN message can not be received.");
+                }
+            } catch (SQLException sqle) {
+                throw new DSpaceBadRequestException("Notify Service [" + notification.getOrigin()
+                + "] unknown. LDN message can not be received.");
+            }
+        }
+    }
 }

dspace-server-webapp/src/test/java/org/dspace/app/rest/ItemFilterRestRepositoryIT.java

@@ -75,30 +75,14 @@ public class ItemFilterRestRepositoryIT extends AbstractControllerIntegrationTes
             .perform(get("/api/config/itemfilters")
                 .param("size", "30"))
             .andExpect(status().isOk())
-            .andExpect(jsonPath("$.page.totalElements", is(21)))
+            .andExpect(jsonPath("$.page.totalElements", is(5)))
             .andExpect(jsonPath("$.page.totalPages", is(1)))
             .andExpect(jsonPath("$.page.size", is(30)))
             .andExpect(jsonPath("$._embedded.itemfilters", contains(
-                hasJsonPath("$.id", is("a-common-or_statement")),
                 hasJsonPath("$.id", is("always_true_filter")),
-                hasJsonPath("$.id", is("dc-identifier-uri-contains-doi_condition")),
                 hasJsonPath("$.id", is("demo_filter")),
                 hasJsonPath("$.id", is("doi-filter")),
-                hasJsonPath("$.id", is("driver-document-type_condition")),
-                hasJsonPath("$.id", is("example-doi_filter")),
-                hasJsonPath("$.id", is("has-at-least-one-bitstream_condition")),
-                hasJsonPath("$.id", is("has-bitstream_filter")),
-                hasJsonPath("$.id", is("has-one-bitstream_condition")),
                 hasJsonPath("$.id", is("in-outfit-collection_condition")),
-                hasJsonPath("$.id", is("is-archived_condition")),
-                hasJsonPath("$.id", is("is-withdrawn_condition")),
-                hasJsonPath("$.id", is("item-is-public_condition")),
-                hasJsonPath("$.id", is("openaire_filter")),
-                hasJsonPath("$.id", is("simple-demo_filter")),
-                hasJsonPath("$.id", is("title-contains-demo_condition")),
-                hasJsonPath("$.id", is("title-starts-with-pattern_condition")),
-                hasJsonPath("$.id", is("type-equals-dataset_condition")),
-                hasJsonPath("$.id", is("type-equals-journal-article_condition")),
                 hasJsonPath("$.id", is("type_filter")))));
     }
 }

dspace-server-webapp/src/test/java/org/dspace/app/rest/LDNInboxControllerIT.java

@@ -303,14 +303,10 @@ public class LDNInboxControllerIT extends AbstractControllerIntegrationTest {
             .perform(post("/ldn/inbox")
                 .contentType("application/ld+json")
                 .content(message))
-            .andExpect(status().isAccepted());
+            .andExpect(status().isBadRequest());
 
         int processed = ldnMessageService.extractAndProcessMessageFromQueue(context);
         assertEquals(processed, 0);
-
-        LDNMessageEntity ldnMessage = ldnMessageService.find(context, notification.getId());
-        checkStoredLDNMessage(notification, ldnMessage, object);
-        assertEquals(ldnMessage.getQueueStatus(), LDNMessageEntity.QUEUE_STATUS_UNTRUSTED_IP);
     }
 
     @Test
@@ -343,14 +339,80 @@ public class LDNInboxControllerIT extends AbstractControllerIntegrationTest {
             .perform(post("/ldn/inbox")
                 .contentType("application/ld+json")
                 .content(message))
-            .andExpect(status().isAccepted());
+            .andExpect(status().isBadRequest());
 
         int processed = ldnMessageService.extractAndProcessMessageFromQueue(context);
         assertEquals(processed, 0);
 
-        LDNMessageEntity ldnMessage = ldnMessageService.find(context, notification.getId());
+    }
-        checkStoredLDNMessage(notification, ldnMessage, object);
+
-        assertEquals(ldnMessage.getQueueStatus(), LDNMessageEntity.QUEUE_STATUS_UNTRUSTED);
+    @Test
+    public void ldnInboxOutOfRangeIPTest() throws Exception {
+        context.turnOffAuthorisationSystem();
+        Community community = CommunityBuilder.createCommunity(context).withName("community").build();
+        Collection collection = CollectionBuilder.createCollection(context, community).build();
+        Item item = ItemBuilder.createItem(context, collection).build();
+        String object = configurationService.getProperty("dspace.ui.url") + "/handle/" + item.getHandle();
+        NotifyServiceEntity notifyServiceEntity =
+            NotifyServiceBuilder.createNotifyServiceBuilder(context)
+                                .withName("service name")
+                                .withDescription("service description")
+                                .withUrl("service url")
+                                .withLdnUrl("https://overlay-journal.com/inbox/")
+                                .withLowerIp("127.0.0.2")
+                                .withUpperIp("127.0.0.3")
+                                .build();
+        context.restoreAuthSystemState();
+
+        InputStream announceEndorsementStream = getClass().getResourceAsStream("ldn_announce_endorsement.json");
+        String announceEndorsement = IOUtils.toString(announceEndorsementStream, Charset.defaultCharset());
+        announceEndorsementStream.close();
+        String message = announceEndorsement.replaceAll("<<object>>", object);
+        message = message.replaceAll("<<object_handle>>", object);
+
+        ObjectMapper mapper = new ObjectMapper();
+        Notification notification = mapper.readValue(message, Notification.class);
+        getClient()
+            .perform(post("/ldn/inbox")
+                .contentType("application/ld+json")
+                .content(message))
+            .andExpect(status().isBadRequest());
+
+    }
+
+    @Test
+    public void ldnInboxOutOfRangeIPwithDisabledCheckTest() throws Exception {
+        context.turnOffAuthorisationSystem();
+        Community community = CommunityBuilder.createCommunity(context).withName("community").build();
+        Collection collection = CollectionBuilder.createCollection(context, community).build();
+        Item item = ItemBuilder.createItem(context, collection).build();
+        configurationService.setProperty("ldn.notify.inbox.block-untrusted-ip", false);
+        String object = configurationService.getProperty("dspace.ui.url") + "/handle/" + item.getHandle();
+        NotifyServiceEntity notifyServiceEntity =
+            NotifyServiceBuilder.createNotifyServiceBuilder(context)
+                                .withName("service name")
+                                .withDescription("service description")
+                                .withUrl("service url")
+                                .withLdnUrl("https://overlay-journal.com/inbox/")
+                                .withLowerIp("127.0.0.2")
+                                .withUpperIp("127.0.0.3")
+                                .build();
+        context.restoreAuthSystemState();
+
+        InputStream announceEndorsementStream = getClass().getResourceAsStream("ldn_announce_endorsement.json");
+        String announceEndorsement = IOUtils.toString(announceEndorsementStream, Charset.defaultCharset());
+        announceEndorsementStream.close();
+        String message = announceEndorsement.replaceAll("<<object>>", object);
+        message = message.replaceAll("<<object_handle>>", object);
+
+        ObjectMapper mapper = new ObjectMapper();
+        Notification notification = mapper.readValue(message, Notification.class);
+        getClient()
+            .perform(post("/ldn/inbox")
+                .contentType("application/ld+json")
+                .content(message))
+            .andExpect(status().isAccepted());
+
     }
 
     @Override

dspace-server-webapp/src/test/resources/org/dspace/app/rest/ldn_offer_review.json

@@ -24,7 +24,7 @@
   },
   "origin": {
     "id": "https://research-organisation.org/repository",
-    "inbox": "sookah",
+    "inbox": "https://review-service.com/inbox/",
     "type": "Service"
   },
   "target": {

dspace-server-webapp/src/test/resources/org/dspace/app/rest/ldn_offer_review2.json

@@ -24,7 +24,7 @@
   },
   "origin": {
     "id": "https://research-organisation.org/repository",
-    "inbox": "sookah",
+    "inbox": "https://review-service.com/inbox/",
     "type": "Service"
   },
   "target": {

dspace/config/dspace.cfg

@@ -1674,4 +1674,4 @@ include = ${module_dir}/usage-statistics.cfg
 include = ${module_dir}/versioning.cfg
 include = ${module_dir}/workflow.cfg
 include = ${module_dir}/external-providers.cfg
-include = ${module_dir}/coar-notify-ldn.cfg
+include = ${module_dir}/ldn.cfg

dspace/config/item-submission.xml

@@ -285,7 +285,7 @@
             <!-- <step id="sherpaPolicies"/> -->
 
             <!--Step will be COAR Notify services to the item -->
-            <step id="coarnotify"/>
+            <!-- <step id="coarnotify"/> -->
 
             <!--Step will be to Upload the item -->
             <step id="upload"/>

dspace/config/modules/ldn.cfg

@@ -5,12 +5,13 @@
 #---------------------------------------------------------------#
 
 
-# For debugging purposes only, skip the check on the IP range.
+# check on the IP number on incoming LDN Messages against the IP Range configured
-#coar-notify.ip-range.enabled = false
+# on the Notify Service known and found as the message sender
+# ldn.ip-range.enabled = false
 
 #### LDN CONFIGURATION ####
 # To enable the LDN service, set to true.
-ldn.enabled = true
+ldn.enabled = false
 
 #LDN message inbox endpoint
 ldn.notify.inbox = ${dspace.server.url}/ldn/inbox
@@ -37,6 +38,13 @@ ldn.processor.max.attempts = 5
 # a new timeout, such as: new_timeout = now + ldn.processor.queue.msg.timeout (in minutes)
 ldn.processor.queue.msg.timeout = 60
 
+# Blocks the storage of incoming LDN messages with unknown Notify Service (origin)
+ldn.notify.inbox.block-untrusted = true
+
+# Blocks the storage of incoming LDN messages with known Notify Service (origin)
+# and out-of-range IP
+ldn.notify.inbox.block-untrusted-ip = true
+
 
 # EMAIL CONFIGURATION
 

dspace/config/spring/api/coar-notify.xml

@@ -1,37 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<beans xmlns="http://www.springframework.org/schema/beans" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-       xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd"
-       default-lazy-init="true">
-
-    <bean class="org.dspace.coarnotify.SubmissionNotifyServiceImpl"/>
-
-    <bean class="org.dspace.coarnotify.NotifyConfigurationService">
-        <property name="patterns">
-            <map>
-                <entry key="coarnotify">
-                    <list>
-                        <ref bean="requestReview"/>
-                        <ref bean="requestEndorsement"/>
-                        <ref bean="requestIngest" />
-                    </list>
-                </entry>
-            </map>
-        </property>
-    </bean>
-
-    <bean id="requestReview" class="org.dspace.coarnotify.NotifyPattern">
-        <property name="pattern" value="request-review"/>
-        <property name="multipleRequest" value="true"/>
-    </bean>
-
-    <bean id="requestEndorsement" class="org.dspace.coarnotify.NotifyPattern">
-        <property name="pattern" value="request-endorsement"/>
-        <property name="multipleRequest" value="true"/>
-    </bean>
-
-    <bean id="requestIngest" class="org.dspace.coarnotify.NotifyPattern">
-        <property name="pattern" value="request-ingest"/>
-        <property name="multipleRequest" value="false"/>
-    </bean>
-
-</beans>

dspace/config/spring/api/item-filters.xml

@@ -346,4 +346,9 @@
         </property>
     </bean>
 
+    <util:map id="ldnItemFilters" key-type="java.lang.String"
+              value-type="org.dspace.content.logic.LogicalStatement">
+       <entry key="type_filter" value-ref="type_filter"/>
+       <entry key="doi-filter" value-ref="doi-filter"/>
+	</util:map>
 </beans>

dspace/config/spring/api/ldn-coar-notify.xml

@@ -289,4 +289,35 @@
         </property>
     </bean>
 
+    <bean class="org.dspace.coarnotify.SubmissionNotifyServiceImpl"/>
+
+    <bean class="org.dspace.coarnotify.NotifyConfigurationService">
+        <property name="patterns">
+            <map>
+                <entry key="coarnotify">
+                    <list>
+                        <ref bean="requestReview"/>
+                        <ref bean="requestEndorsement"/>
+                        <ref bean="requestIngest" />
+                    </list>
+                </entry>
+            </map>
+        </property>
+    </bean>
+
+    <bean id="requestReview" class="org.dspace.coarnotify.NotifyPattern">
+        <property name="pattern" value="request-review"/>
+        <property name="multipleRequest" value="true"/>
+    </bean>
+
+    <bean id="requestEndorsement" class="org.dspace.coarnotify.NotifyPattern">
+        <property name="pattern" value="request-endorsement"/>
+        <property name="multipleRequest" value="true"/>
+    </bean>
+
+    <bean id="requestIngest" class="org.dspace.coarnotify.NotifyPattern">
+        <property name="pattern" value="request-ingest"/>
+        <property name="multipleRequest" value="false"/>
+    </bean>
+
 </beans>