Merge pull request #7987 from mwoodiupui/7986

Use a connection pool for communication with Solr

dspace-api/pom.xml

@@ -903,14 +903,12 @@
         <dependency>
             <groupId>org.apache.velocity</groupId>
             <artifactId>velocity-engine-core</artifactId>
-            <version>2.0</version>
             <type>jar</type>
         </dependency>
 
         <dependency>
             <groupId>org.xmlunit</groupId>
             <artifactId>xmlunit-core</artifactId>
-            <version>2.6.3</version>
             <scope>test</scope>
         </dependency>
 
@@ -934,6 +932,81 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>org.mock-server</groupId>
+            <artifactId>mockserver-junit-rule</artifactId>
+            <version>5.11.2</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
+    <dependencyManagement>
+        <dependencies>
+          <!-- for mockserver -->
+          <!-- Solve dependency convergence issues related to
+               'mockserver-junit-rule' by selecting the versions we want to use. -->
+          <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-text</artifactId>
+            <version>1.9</version>
+          </dependency>
+          <dependency>
+            <groupId>io.netty</groupId>
+            <artifactId>netty-buffer</artifactId>
+            <version>4.1.53.Final</version>
+          </dependency>
+          <dependency>
+            <groupId>io.netty</groupId>
+            <artifactId>netty-transport</artifactId>
+            <version>4.1.53.Final</version>
+          </dependency>
+          <dependency>
+            <groupId>io.netty</groupId>
+            <artifactId>netty-common</artifactId>
+            <version>4.1.53.Final</version>
+          </dependency>
+          <dependency>
+            <groupId>io.netty</groupId>
+            <artifactId>netty-handler</artifactId>
+            <version>4.1.53.Final</version>
+          </dependency>
+          <dependency>
+            <groupId>io.netty</groupId>
+            <artifactId>netty-codec</artifactId>
+            <version>4.1.53.Final</version>
+          </dependency>
+          <dependency>
+            <groupId>org.apache.velocity</groupId>
+            <artifactId>velocity-engine-core</artifactId>
+            <version>2.2</version>
+          </dependency>
+          <dependency>
+              <groupId>org.xmlunit</groupId>
+              <artifactId>xmlunit-core</artifactId>
+              <version>2.8.0</version>
+              <scope>test</scope>
+          </dependency>
+          <dependency>
+            <groupId>com.github.java-json-tools</groupId>
+            <artifactId>json-schema-validator</artifactId>
+            <version>2.2.14</version>
+          </dependency>
+          <dependency>
+            <groupId>jakarta.xml.bind</groupId>
+            <artifactId>jakarta.xml.bind-api</artifactId>
+            <version>2.3.3</version>
+          </dependency>
+          <dependency>
+            <groupId>javax.validation</groupId>
+            <artifactId>validation-api</artifactId>
+            <version>2.0.1.Final</version>
+          </dependency>
+          <dependency>
+            <groupId>io.swagger</groupId>
+            <artifactId>swagger-core</artifactId>
+            <version>1.6.2</version>
+          </dependency>
+        </dependencies>
+    </dependencyManagement>
+
 </project>

dspace-api/src/main/java/org/dspace/authority/AuthoritySolrServiceImpl.java

@@ -11,6 +11,8 @@ import java.io.IOException;
 import java.net.MalformedURLException;
 import java.util.ArrayList;
 import java.util.List;
+import javax.inject.Inject;
+import javax.inject.Named;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -22,6 +24,7 @@ import org.apache.solr.client.solrj.response.FacetField;
 import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.common.SolrInputDocument;
 import org.dspace.authority.indexer.AuthorityIndexingService;
+import org.dspace.service.impl.HttpConnectionPoolService;
 import org.dspace.services.ConfigurationService;
 import org.dspace.services.factory.DSpaceServicesFactory;
 
@@ -35,6 +38,9 @@ public class AuthoritySolrServiceImpl implements AuthorityIndexingService, Autho
 
     private static final Logger log = LogManager.getLogger(AuthoritySolrServiceImpl.class);
 
+    @Inject @Named("solrHttpConnectionPoolService")
+    private HttpConnectionPoolService httpConnectionPoolService;
+
     protected AuthoritySolrServiceImpl() {
 
     }
@@ -54,7 +60,9 @@ public class AuthoritySolrServiceImpl implements AuthorityIndexingService, Autho
 
             log.debug("Solr authority URL: " + solrService);
 
-            HttpSolrClient solrServer = new HttpSolrClient.Builder(solrService).build();
+            HttpSolrClient solrServer = new HttpSolrClient.Builder(solrService)
+                    .withHttpClient(httpConnectionPoolService.getClient())
+                    .build();
             solrServer.setBaseURL(solrService);
 
             SolrQuery solrQuery = new SolrQuery().setQuery("*:*");

dspace-api/src/main/java/org/dspace/discovery/SolrSearchCore.java

@@ -8,6 +8,7 @@
 package org.dspace.discovery;
 
 import java.io.IOException;
+import javax.inject.Named;
 
 import org.apache.commons.validator.routines.UrlValidator;
 import org.apache.logging.log4j.LogManager;
@@ -18,13 +19,14 @@ import org.apache.solr.client.solrj.SolrRequest;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.impl.HttpSolrClient;
 import org.dspace.discovery.indexobject.IndexableItem;
+import org.dspace.service.impl.HttpConnectionPoolService;
 import org.dspace.services.ConfigurationService;
 import org.dspace.services.factory.DSpaceServicesFactory;
 import org.dspace.storage.rdbms.DatabaseUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 
 /**
- * Bean containing the SolrClient for the search core
+ * Bean containing the SolrClient for the search core.
  * @author Kevin Van de Velde (kevin at atmire dot com)
  */
 public class SolrSearchCore {
@@ -34,6 +36,8 @@ public class SolrSearchCore {
     protected IndexingService indexingService;
     @Autowired
     protected ConfigurationService configurationService;
+    @Autowired @Named("solrHttpConnectionPoolService")
+    protected HttpConnectionPoolService httpConnectionPoolService;
 
     /**
      *  SolrServer for processing indexing events.
@@ -79,7 +83,9 @@ public class SolrSearchCore {
                 .getBooleanProperty("discovery.solr.url.validation.enabled", true)) {
                 try {
                     log.debug("Solr URL: " + solrService);
-                    HttpSolrClient solrServer = new HttpSolrClient.Builder(solrService).build();
+                    HttpSolrClient solrServer = new HttpSolrClient.Builder(solrService)
+                            .withHttpClient(httpConnectionPoolService.getClient())
+                            .build();
 
                     solrServer.setBaseURL(solrService);
                     solrServer.setUseMultiPartPost(true);

dspace-api/src/main/java/org/dspace/service/impl/HttpConnectionPoolService.java

@@ -0,0 +1,196 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE and NOTICE files at the root of the source
+ * tree and available online at
+ *
+ * http://www.dspace.org/license/
+ */
+package org.dspace.service.impl;
+
+import java.util.concurrent.TimeUnit;
+import javax.annotation.PostConstruct;
+import javax.inject.Inject;
+import javax.inject.Named;
+import javax.inject.Singleton;
+
+import org.apache.http.HeaderElement;
+import org.apache.http.HeaderElementIterator;
+import org.apache.http.HttpResponse;
+import org.apache.http.conn.ConnectionKeepAliveStrategy;
+import org.apache.http.conn.HttpClientConnectionManager;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.message.BasicHeaderElementIterator;
+import org.apache.http.protocol.HTTP;
+import org.apache.http.protocol.HttpContext;
+import org.dspace.services.ConfigurationService;
+
+/**
+ * Factory for HTTP clients sharing a pool of connections.
+ *
+ * <p>You may create multiple pools.  Each is identified by a configuration
+ * "prefix" (passed to the constructor) which is used to create names of
+ * properties which will configure the pool.  The properties are:
+ *
+ * <dl>
+ *   <dt>PREFIX.client.keepAlive</dt>
+ *   <dd>Default keep-alive time for open connections, in milliseconds</dd>
+ *   <dt>PREFIX.client.maxTotalConnections</dt>
+ *   <dd>maximum open connections</dd>
+ *   <dt>PREFIX.client.maxPerRoute</dt>
+ *   <dd>maximum open connections per service instance</dd>
+ *   <dt>PREFIX.client.timeToLive</dt>
+ *   <dd>maximum lifetime of a pooled connection, in seconds</dd>
+ * </dl>
+ *
+ * @author Mark H. Wood <mwood@iupui.edu>
+ */
+@Named
+@Singleton
+public class HttpConnectionPoolService {
+    @Inject
+    ConfigurationService configurationService;
+
+    /** Configuration properties will begin with this string. */
+    private final String configPrefix;
+
+    /** Maximum number of concurrent pooled connections. */
+    private static final int DEFAULT_MAX_TOTAL_CONNECTIONS = 20;
+
+    /** Maximum number of concurrent pooled connections per route. */
+    private static final int DEFAULT_MAX_PER_ROUTE = 15;
+
+    /** Keep connections open at least this long, if the response did not
+     *  specify:  milliseconds
+     */
+    private static final int DEFAULT_KEEPALIVE = 5 * 1000;
+
+    /** Pooled connection maximum lifetime:  seconds */
+    private static final int DEFAULT_TTL = 10 * 60;
+
+    /** Clean up stale connections this often:  milliseconds */
+    private static final int CHECK_INTERVAL = 1000;
+
+    /** Connection idle if unused for this long:  seconds */
+    private static final int IDLE_INTERVAL = 30;
+
+    private PoolingHttpClientConnectionManager connManager;
+
+    private final ConnectionKeepAliveStrategy keepAliveStrategy
+            = new KeepAliveStrategy();
+
+    /**
+     * Construct a pool for a given set of configuration properties.
+     *
+     * @param configPrefix Configuration property names will begin with this.
+     */
+    public HttpConnectionPoolService(String configPrefix) {
+        this.configPrefix = configPrefix;
+    }
+
+    @PostConstruct
+    protected void init() {
+        connManager = new PoolingHttpClientConnectionManager(
+                configurationService.getIntProperty(configPrefix + ".client.timeToLive", DEFAULT_TTL),
+                TimeUnit.SECONDS);
+
+        connManager.setMaxTotal(configurationService.getIntProperty(
+                configPrefix + ".client.maxTotalConnections", DEFAULT_MAX_TOTAL_CONNECTIONS));
+        connManager.setDefaultMaxPerRoute(
+                configurationService.getIntProperty(configPrefix + ".client.maxPerRoute",
+                        DEFAULT_MAX_PER_ROUTE));
+
+        Thread connectionMonitor = new IdleConnectionMonitorThread(connManager);
+        connectionMonitor.setDaemon(true);
+        connectionMonitor.start();
+    }
+
+    /**
+     * Create an HTTP client which uses a pooled connection.
+     *
+     * @return the client.
+     */
+    public CloseableHttpClient getClient() {
+        CloseableHttpClient httpClient = HttpClientBuilder.create()
+                .setKeepAliveStrategy(keepAliveStrategy)
+                .setConnectionManager(connManager)
+                .build();
+        return httpClient;
+    }
+
+    /**
+     * A connection keep-alive strategy that obeys the Keep-Alive header and
+     * applies a default if none is given.
+     *
+     * Swiped from https://www.baeldung.com/httpclient-connection-management
+     */
+    public class KeepAliveStrategy
+            implements ConnectionKeepAliveStrategy {
+        @Override
+        public long getKeepAliveDuration(HttpResponse response,
+                HttpContext context) {
+            HeaderElementIterator it = new BasicHeaderElementIterator(
+                    response.headerIterator(HTTP.CONN_KEEP_ALIVE));
+            while (it.hasNext()) {
+                HeaderElement he = it.nextElement();
+                String name = he.getName();
+                String value = he.getValue();
+                if (value != null && "timeout".equalsIgnoreCase(name)) {
+                    return Long.parseLong(value) * 1000;
+                }
+            }
+
+            // If server did not request keep-alive, use configured value.
+            return configurationService.getIntProperty(configPrefix + ".client.keepAlive",
+                    DEFAULT_KEEPALIVE);
+        }
+    }
+
+    /**
+     * Clean up stale connections.
+     *
+     * Swiped from https://www.baeldung.com/httpclient-connection-management
+     */
+    public class IdleConnectionMonitorThread
+            extends Thread {
+        private final HttpClientConnectionManager connMgr;
+        private volatile boolean shutdown;
+
+        /**
+         * Constructor.
+         *
+         * @param connMgr the manager to be monitored.
+         */
+        public IdleConnectionMonitorThread(
+            PoolingHttpClientConnectionManager connMgr) {
+            super();
+            this.connMgr = connMgr;
+        }
+
+        @Override
+        public void run() {
+            try {
+                while (!shutdown) {
+                    synchronized (this) {
+                        wait(CHECK_INTERVAL);
+                        connMgr.closeExpiredConnections();
+                        connMgr.closeIdleConnections(IDLE_INTERVAL, TimeUnit.SECONDS);
+                    }
+                }
+            } catch (InterruptedException ex) {
+                shutdown();
+            }
+        }
+
+        /**
+         * Cause a controlled exit from the thread.
+         */
+        public void shutdown() {
+            shutdown = true;
+            synchronized (this) {
+                notifyAll();
+            }
+        }
+    }
+}

dspace-api/src/main/java/org/dspace/statistics/SolrStatisticsCore.java

@@ -9,9 +9,12 @@ package org.dspace.statistics;
 
 import static org.apache.logging.log4j.LogManager.getLogger;
 
+import javax.inject.Named;
+
 import org.apache.logging.log4j.Logger;
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.dspace.service.impl.HttpConnectionPoolService;
 import org.dspace.services.ConfigurationService;
 import org.springframework.beans.factory.annotation.Autowired;
 
@@ -20,13 +23,16 @@ import org.springframework.beans.factory.annotation.Autowired;
  */
 public class SolrStatisticsCore {
 
-    private static Logger log = getLogger(SolrStatisticsCore.class);
+    private static final Logger log = getLogger();
 
     protected SolrClient solr = null;
 
     @Autowired
     private ConfigurationService configurationService;
 
+    @Autowired @Named("solrHttpConnectionPoolService")
+    private HttpConnectionPoolService httpConnectionPoolService;
+
     /**
      * Returns the {@link SolrClient} for the Statistics core.
      * Initializes it if needed.
@@ -50,7 +56,9 @@ public class SolrStatisticsCore {
         log.info("usage-statistics.dbfile:  {}", configurationService.getProperty("usage-statistics.dbfile"));
 
         try {
-            solr = new HttpSolrClient.Builder(solrService).build();
+            solr = new HttpSolrClient.Builder(solrService)
+                    .withHttpClient(httpConnectionPoolService.getClient())
+                    .build();
         } catch (Exception e) {
             log.error("Error accessing Solr server configured in 'solr-statistics.server'", e);
         }

dspace-api/src/test/java/org/dspace/service/impl/HttpConnectionPoolServiceTest.java

@@ -0,0 +1,96 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE and NOTICE files at the root of the source
+ * tree and available online at
+ *
+ * http://www.dspace.org/license/
+ */
+package org.dspace.service.impl;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockserver.model.HttpRequest.request;
+import static org.mockserver.model.HttpResponse.response;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.client.methods.RequestBuilder;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.dspace.AbstractDSpaceTest;
+import org.dspace.services.ConfigurationService;
+import org.dspace.services.factory.DSpaceServicesFactory;
+import org.eclipse.jetty.http.HttpStatus;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockserver.client.MockServerClient;
+import org.mockserver.junit.MockServerRule;
+
+/**
+ *
+ * @author Mark H. Wood <mwood@iupui.edu>
+ */
+public class HttpConnectionPoolServiceTest
+        extends AbstractDSpaceTest {
+    private static ConfigurationService configurationService;
+
+    @Rule
+    public MockServerRule mockServerRule = new MockServerRule(this);
+
+    private MockServerClient mockServerClient;
+
+    @BeforeClass
+    public static void initClass() {
+        configurationService = DSpaceServicesFactory.getInstance()
+                .getConfigurationService();
+    }
+
+    /**
+     * Test of getClient method, of class HttpConnectionPoolService.
+     * @throws java.io.IOException if a connection cannot be closed.
+     * @throws java.net.URISyntaxException when an invalid URI is constructed.
+     */
+    @Test
+    public void testGetClient()
+            throws IOException, URISyntaxException {
+        System.out.println("getClient");
+
+        configurationService.setProperty("solr.client.maxTotalConnections", 2);
+        configurationService.setProperty("solr.client.maxPerRoute", 2);
+        HttpConnectionPoolService instance = new HttpConnectionPoolService("solr");
+        instance.configurationService = configurationService;
+        instance.init();
+
+        final String testPath = "/test";
+        mockServerClient.when(
+                request()
+                .withPath(testPath)
+        ).respond(
+                response()
+                .withStatusCode(HttpStatus.OK_200)
+        );
+
+        try (CloseableHttpClient httpClient = instance.getClient()) {
+            assertNotNull("getClient should always return a client", httpClient);
+
+            URI uri = new URIBuilder()
+                    .setScheme("http")
+                    .setHost("localhost")
+                    .setPort(mockServerClient.getPort())
+                    .setPath(testPath)
+                    .build();
+            System.out.println(uri.toString());
+            HttpUriRequest request = RequestBuilder.get(uri)
+                    .build();
+            try (CloseableHttpResponse response = httpClient.execute(request)) {
+                assertEquals("Response status should be OK", HttpStatus.OK_200,
+                        response.getStatusLine().getStatusCode());
+            }
+        }
+    }
+}

dspace-oai/src/main/java/org/dspace/xoai/services/impl/solr/DSpaceSolrServerResolver.java

@@ -7,11 +7,14 @@
  */
 package org.dspace.xoai.services.impl.solr;
 
+import javax.inject.Named;
+
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.dspace.service.impl.HttpConnectionPoolService;
 import org.dspace.xoai.services.api.config.ConfigurationService;
 import org.dspace.xoai.services.api.solr.SolrServerResolver;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -23,12 +26,17 @@ public class DSpaceSolrServerResolver implements SolrServerResolver {
     @Autowired
     private ConfigurationService configurationService;
 
+    @Autowired @Named("solrHttpConnectionPoolService")
+    private HttpConnectionPoolService httpConnectionPoolService;
+
     @Override
     public SolrClient getServer() throws SolrServerException {
         if (server == null) {
             String serverUrl = configurationService.getProperty("oai.solr.url");
             try {
-                server = new HttpSolrClient.Builder(serverUrl).build();
+                server = new HttpSolrClient.Builder(serverUrl)
+                        .withHttpClient(httpConnectionPoolService.getClient())
+                        .build();
                 log.debug("OAI Solr Server Initialized");
             } catch (Exception e) {
                 log.error("Could not initialize OAI Solr Server at " + serverUrl , e);

dspace-oai/src/main/java/org/dspace/xoai/solr/DSpaceSolrServer.java

@@ -13,6 +13,7 @@ import org.apache.logging.log4j.Logger;
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.dspace.service.impl.HttpConnectionPoolService;
 import org.dspace.services.ConfigurationService;
 import org.dspace.services.factory.DSpaceServicesFactory;
 
@@ -33,9 +34,16 @@ public class DSpaceSolrServer {
         if (_server == null) {
             ConfigurationService configurationService
                     = DSpaceServicesFactory.getInstance().getConfigurationService();
+            HttpConnectionPoolService httpConnectionPoolService
+                    = DSpaceServicesFactory.getInstance()
+                            .getServiceManager()
+                            .getServiceByName("solrHttpConnectionPoolService",
+                                    HttpConnectionPoolService.class);
             String serverUrl = configurationService.getProperty("oai.solr.url");
             try {
-                _server = new HttpSolrClient.Builder(serverUrl).build();
+                _server = new HttpSolrClient.Builder(serverUrl)
+                        .withHttpClient(httpConnectionPoolService.getClient())
+                        .build();
                 log.debug("OAI Solr Server Initialized");
             } catch (Exception e) {
                 log.error("Could not initialize OAI Solr Server at " + serverUrl , e);

dspace/config/dspace.cfg

@@ -45,6 +45,22 @@ default.language = en_US
 # Since DSpace 7, SOLR must be installed as a stand-alone service
 solr.server = http://localhost:8983/solr
 
+# Solr connection pool.
+# If you change these values, the changes are not effective until DSpace is
+# restarted.
+#
+# Maximum open connections to Solr:
+# solr.client.maxTotalConnections = 20
+#
+# Maximum open connections per Solr instance:
+# solr.client.maxPerRoute = 15
+#
+# Default keep-alive time for open Solr connections, in milliseconds:
+# solr.client.keepAlive = 5000
+#
+# Maximum lifetime of a pooled connection, in seconds:
+# solr.client.timeToLive = 600
+
 ##### Database settings #####
 # DSpace only supports two database types: PostgreSQL or Oracle
 

dspace/config/spring/api/core-services.xml

@@ -63,6 +63,13 @@
     <bean class="org.dspace.content.authority.ChoiceAuthorityServiceImpl"/>
     <bean class="org.dspace.content.authority.MetadataAuthorityServiceImpl" lazy-init="true"/>
 
+    <bean class='org.dspace.service.impl.HttpConnectionPoolService'
+          id='solrHttpConnectionPoolService'
+          scope='singleton'
+          autowire-candidate='true'>
+        <constructor-arg name='configPrefix' value='solr'/>
+    </bean>
+
     <!-- Ensure PluginService is initialized properly via init() method -->
     <bean class="org.dspace.core.LegacyPluginServiceImpl" init-method="init"/>
     <bean class="org.dspace.core.LicenseServiceImpl"/>