hazza/DSpace
1
0
Fork 0
mirror of https://github.com/DSpace/DSpace.git synced 2025-10-07 01:54:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #7987 from mwoodiupui/7986

Browse Source 
Use a connection pool for communication with Solr
This commit is contained in:
Tim Donohue
2022-01-11 16:16:42 -06:00
committed by GitHub
parent c6b6ca6c0f edba8a2811
commit 611b015e45
10 changed files with 435 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
dspace-api/pom.xml
View File

@@ -903,14 +903,12 @@
<dependency>
<groupId>org.apache.velocity</groupId>
<artifactId>velocity-engine-core</artifactId>
<version>2.0</version>
<type>jar</type>
</dependency>
<dependency>
<groupId>org.xmlunit</groupId>
<artifactId>xmlunit-core</artifactId>
<version>2.6.3</version>
<scope>test</scope>
</dependency>
@@ -934,6 +932,81 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mock-server</groupId>
<artifactId>mockserver-junit-rule</artifactId>
<version>5.11.2</version>
<scope>test</scope>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<!-- for mockserver -->
<!-- Solve dependency convergence issues related to
'mockserver-junit-rule' by selecting the versions we want to use. -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-text</artifactId>
<version>1.9</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-buffer</artifactId>
<version>4.1.53.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport</artifactId>
<version>4.1.53.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
<version>4.1.53.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-handler</artifactId>
<version>4.1.53.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec</artifactId>
<version>4.1.53.Final</version>
</dependency>
<dependency>
<groupId>org.apache.velocity</groupId>
<artifactId>velocity-engine-core</artifactId>
<version>2.2</version>
</dependency>
<dependency>
<groupId>org.xmlunit</groupId>
<artifactId>xmlunit-core</artifactId>
<version>2.8.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.github.java-json-tools</groupId>
<artifactId>json-schema-validator</artifactId>
<version>2.2.14</version>
</dependency>
<dependency>
<groupId>jakarta.xml.bind</groupId>
<artifactId>jakarta.xml.bind-api</artifactId>
<version>2.3.3</version>
</dependency>
<dependency>
<groupId>javax.validation</groupId>
<artifactId>validation-api</artifactId>
<version>2.0.1.Final</version>
</dependency>
<dependency>
<groupId>io.swagger</groupId>
<artifactId>swagger-core</artifactId>
<version>1.6.2</version>
</dependency>
</dependencies>
</dependencyManagement>
</project>

10
dspace-api/src/main/java/org/dspace/authority/AuthoritySolrServiceImpl.java
View File

@@ -11,6 +11,8 @@ import java.io.IOException;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.List;
import javax.inject.Inject;
import javax.inject.Named;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -22,6 +24,7 @@ import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrInputDocument;
import org.dspace.authority.indexer.AuthorityIndexingService;
import org.dspace.service.impl.HttpConnectionPoolService;
import org.dspace.services.ConfigurationService;
import org.dspace.services.factory.DSpaceServicesFactory;
@@ -35,6 +38,9 @@ public class AuthoritySolrServiceImpl implements AuthorityIndexingService, Autho
private static final Logger log = LogManager.getLogger(AuthoritySolrServiceImpl.class);
@Inject @Named("solrHttpConnectionPoolService")
private HttpConnectionPoolService httpConnectionPoolService;
protected AuthoritySolrServiceImpl() {
}
@@ -54,7 +60,9 @@ public class AuthoritySolrServiceImpl implements AuthorityIndexingService, Autho
log.debug("Solr authority URL: " + solrService);
HttpSolrClient solrServer = new HttpSolrClient.Builder(solrService).build();
HttpSolrClient solrServer = new HttpSolrClient.Builder(solrService)
.withHttpClient(httpConnectionPoolService.getClient())
.build();
solrServer.setBaseURL(solrService);
SolrQuery solrQuery = new SolrQuery().setQuery("*:*");

10
dspace-api/src/main/java/org/dspace/discovery/SolrSearchCore.java
View File

@@ -8,6 +8,7 @@
package org.dspace.discovery;
import java.io.IOException;
import javax.inject.Named;
import org.apache.commons.validator.routines.UrlValidator;
import org.apache.logging.log4j.LogManager;
@@ -18,13 +19,14 @@ import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.dspace.discovery.indexobject.IndexableItem;
import org.dspace.service.impl.HttpConnectionPoolService;
import org.dspace.services.ConfigurationService;
import org.dspace.services.factory.DSpaceServicesFactory;
import org.dspace.storage.rdbms.DatabaseUtils;
import org.springframework.beans.factory.annotation.Autowired;
/**
* Bean containing the SolrClient for the search core
* Bean containing the SolrClient for the search core.
* @author Kevin Van de Velde (kevin at atmire dot com)
*/
public class SolrSearchCore {
@@ -34,6 +36,8 @@ public class SolrSearchCore {
protected IndexingService indexingService;
@Autowired
protected ConfigurationService configurationService;
@Autowired @Named("solrHttpConnectionPoolService")
protected HttpConnectionPoolService httpConnectionPoolService;
/**
* SolrServer for processing indexing events.
@@ -79,7 +83,9 @@ public class SolrSearchCore {
.getBooleanProperty("discovery.solr.url.validation.enabled", true)) {
try {
log.debug("Solr URL: " + solrService);
HttpSolrClient solrServer = new HttpSolrClient.Builder(solrService).build();
HttpSolrClient solrServer = new HttpSolrClient.Builder(solrService)
.withHttpClient(httpConnectionPoolService.getClient())
.build();
solrServer.setBaseURL(solrService);
solrServer.setUseMultiPartPost(true);

196
dspace-api/src/main/java/org/dspace/service/impl/HttpConnectionPoolService.java Normal file
View File

@@ -0,0 +1,196 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.service.impl;
import java.util.concurrent.TimeUnit;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
import javax.inject.Named;
import javax.inject.Singleton;
import org.apache.http.HeaderElement;
import org.apache.http.HeaderElementIterator;
import org.apache.http.HttpResponse;
import org.apache.http.conn.ConnectionKeepAliveStrategy;
import org.apache.http.conn.HttpClientConnectionManager;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicHeaderElementIterator;
import org.apache.http.protocol.HTTP;
import org.apache.http.protocol.HttpContext;
import org.dspace.services.ConfigurationService;
/**
* Factory for HTTP clients sharing a pool of connections.
*
* <p>You may create multiple pools. Each is identified by a configuration
* "prefix" (passed to the constructor) which is used to create names of
* properties which will configure the pool. The properties are:
*
* <dl>
* <dt>PREFIX.client.keepAlive</dt>
* <dd>Default keep-alive time for open connections, in milliseconds</dd>
* <dt>PREFIX.client.maxTotalConnections</dt>
* <dd>maximum open connections</dd>
* <dt>PREFIX.client.maxPerRoute</dt>
* <dd>maximum open connections per service instance</dd>
* <dt>PREFIX.client.timeToLive</dt>
* <dd>maximum lifetime of a pooled connection, in seconds</dd>
* </dl>
*
* @author Mark H. Wood <mwood@iupui.edu>
*/
@Named
@Singleton
public class HttpConnectionPoolService {
@Inject
ConfigurationService configurationService;
/** Configuration properties will begin with this string. */
private final String configPrefix;
/** Maximum number of concurrent pooled connections. */
private static final int DEFAULT_MAX_TOTAL_CONNECTIONS = 20;
/** Maximum number of concurrent pooled connections per route. */
private static final int DEFAULT_MAX_PER_ROUTE = 15;
/** Keep connections open at least this long, if the response did not
* specify: milliseconds
*/
private static final int DEFAULT_KEEPALIVE = 5 * 1000;
/** Pooled connection maximum lifetime: seconds */
private static final int DEFAULT_TTL = 10 * 60;
/** Clean up stale connections this often: milliseconds */
private static final int CHECK_INTERVAL = 1000;
/** Connection idle if unused for this long: seconds */
private static final int IDLE_INTERVAL = 30;
private PoolingHttpClientConnectionManager connManager;
private final ConnectionKeepAliveStrategy keepAliveStrategy
= new KeepAliveStrategy();
/**
* Construct a pool for a given set of configuration properties.
*
* @param configPrefix Configuration property names will begin with this.
*/
public HttpConnectionPoolService(String configPrefix) {
this.configPrefix = configPrefix;
}
@PostConstruct
protected void init() {
connManager = new PoolingHttpClientConnectionManager(
configurationService.getIntProperty(configPrefix + ".client.timeToLive", DEFAULT_TTL),
TimeUnit.SECONDS);
connManager.setMaxTotal(configurationService.getIntProperty(
configPrefix + ".client.maxTotalConnections", DEFAULT_MAX_TOTAL_CONNECTIONS));
connManager.setDefaultMaxPerRoute(
configurationService.getIntProperty(configPrefix + ".client.maxPerRoute",
DEFAULT_MAX_PER_ROUTE));
Thread connectionMonitor = new IdleConnectionMonitorThread(connManager);
connectionMonitor.setDaemon(true);
connectionMonitor.start();
}
/**
* Create an HTTP client which uses a pooled connection.
*
* @return the client.
*/
public CloseableHttpClient getClient() {
CloseableHttpClient httpClient = HttpClientBuilder.create()
.setKeepAliveStrategy(keepAliveStrategy)
.setConnectionManager(connManager)
.build();
return httpClient;
}
/**
* A connection keep-alive strategy that obeys the Keep-Alive header and
* applies a default if none is given.
*
* Swiped from https://www.baeldung.com/httpclient-connection-management
*/
public class KeepAliveStrategy
implements ConnectionKeepAliveStrategy {
@Override
public long getKeepAliveDuration(HttpResponse response,
HttpContext context) {
HeaderElementIterator it = new BasicHeaderElementIterator(
response.headerIterator(HTTP.CONN_KEEP_ALIVE));
while (it.hasNext()) {
HeaderElement he = it.nextElement();
String name = he.getName();
String value = he.getValue();
if (value != null && "timeout".equalsIgnoreCase(name)) {
return Long.parseLong(value) * 1000;
}
}
// If server did not request keep-alive, use configured value.
return configurationService.getIntProperty(configPrefix + ".client.keepAlive",
DEFAULT_KEEPALIVE);
}
}
/**
* Clean up stale connections.
*
* Swiped from https://www.baeldung.com/httpclient-connection-management
*/
public class IdleConnectionMonitorThread
extends Thread {
private final HttpClientConnectionManager connMgr;
private volatile boolean shutdown;
/**
* Constructor.
*
* @param connMgr the manager to be monitored.
*/
public IdleConnectionMonitorThread(
PoolingHttpClientConnectionManager connMgr) {
super();
this.connMgr = connMgr;
}
@Override
public void run() {
try {
while (!shutdown) {
synchronized (this) {
wait(CHECK_INTERVAL);
connMgr.closeExpiredConnections();
connMgr.closeIdleConnections(IDLE_INTERVAL, TimeUnit.SECONDS);
}
}
} catch (InterruptedException ex) {
shutdown();
}
}
/**
* Cause a controlled exit from the thread.
*/
public void shutdown() {
shutdown = true;
synchronized (this) {
notifyAll();
}
}
}
}

12
dspace-api/src/main/java/org/dspace/statistics/SolrStatisticsCore.java
View File

@@ -9,9 +9,12 @@ package org.dspace.statistics;
import static org.apache.logging.log4j.LogManager.getLogger;
import javax.inject.Named;
import org.apache.logging.log4j.Logger;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.dspace.service.impl.HttpConnectionPoolService;
import org.dspace.services.ConfigurationService;
import org.springframework.beans.factory.annotation.Autowired;
@@ -20,13 +23,16 @@ import org.springframework.beans.factory.annotation.Autowired;
*/
public class SolrStatisticsCore {
private static Logger log = getLogger(SolrStatisticsCore.class);
private static final Logger log = getLogger();
protected SolrClient solr = null;
@Autowired
private ConfigurationService configurationService;
@Autowired @Named("solrHttpConnectionPoolService")
private HttpConnectionPoolService httpConnectionPoolService;
/**
* Returns the {@link SolrClient} for the Statistics core.
* Initializes it if needed.
@@ -50,7 +56,9 @@ public class SolrStatisticsCore {
log.info("usage-statistics.dbfile: {}", configurationService.getProperty("usage-statistics.dbfile"));
try {
solr = new HttpSolrClient.Builder(solrService).build();
solr = new HttpSolrClient.Builder(solrService)
.withHttpClient(httpConnectionPoolService.getClient())
.build();
} catch (Exception e) {
log.error("Error accessing Solr server configured in 'solr-statistics.server'", e);
}

96
dspace-api/src/test/java/org/dspace/service/impl/HttpConnectionPoolServiceTest.java Normal file
View File

@@ -0,0 +1,96 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.service.impl;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.mockserver.model.HttpRequest.request;
import static org.mockserver.model.HttpResponse.response;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
import org.dspace.AbstractDSpaceTest;
import org.dspace.services.ConfigurationService;
import org.dspace.services.factory.DSpaceServicesFactory;
import org.eclipse.jetty.http.HttpStatus;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.mockserver.client.MockServerClient;
import org.mockserver.junit.MockServerRule;
/**
*
* @author Mark H. Wood <mwood@iupui.edu>
*/
public class HttpConnectionPoolServiceTest
extends AbstractDSpaceTest {
private static ConfigurationService configurationService;
@Rule
public MockServerRule mockServerRule = new MockServerRule(this);
private MockServerClient mockServerClient;
@BeforeClass
public static void initClass() {
configurationService = DSpaceServicesFactory.getInstance()
.getConfigurationService();
}
/**
* Test of getClient method, of class HttpConnectionPoolService.
* @throws java.io.IOException if a connection cannot be closed.
* @throws java.net.URISyntaxException when an invalid URI is constructed.
*/
@Test
public void testGetClient()
throws IOException, URISyntaxException {
System.out.println("getClient");
configurationService.setProperty("solr.client.maxTotalConnections", 2);
configurationService.setProperty("solr.client.maxPerRoute", 2);
HttpConnectionPoolService instance = new HttpConnectionPoolService("solr");
instance.configurationService = configurationService;
instance.init();
final String testPath = "/test";
mockServerClient.when(
request()
.withPath(testPath)
).respond(
response()
.withStatusCode(HttpStatus.OK_200)
);
try (CloseableHttpClient httpClient = instance.getClient()) {
assertNotNull("getClient should always return a client", httpClient);
URI uri = new URIBuilder()
.setScheme("http")
.setHost("localhost")
.setPort(mockServerClient.getPort())
.setPath(testPath)
.build();
System.out.println(uri.toString());
HttpUriRequest request = RequestBuilder.get(uri)
.build();
try (CloseableHttpResponse response = httpClient.execute(request)) {
assertEquals("Response status should be OK", HttpStatus.OK_200,
response.getStatusLine().getStatusCode());
}
}
}
}

10
dspace-oai/src/main/java/org/dspace/xoai/services/impl/solr/DSpaceSolrServerResolver.java
View File

@@ -7,11 +7,14 @@
*/
package org.dspace.xoai.services.impl.solr;
import javax.inject.Named;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.dspace.service.impl.HttpConnectionPoolService;
import org.dspace.xoai.services.api.config.ConfigurationService;
import org.dspace.xoai.services.api.solr.SolrServerResolver;
import org.springframework.beans.factory.annotation.Autowired;
@@ -23,12 +26,17 @@ public class DSpaceSolrServerResolver implements SolrServerResolver {
@Autowired
private ConfigurationService configurationService;
@Autowired @Named("solrHttpConnectionPoolService")
private HttpConnectionPoolService httpConnectionPoolService;
@Override
public SolrClient getServer() throws SolrServerException {
if (server == null) {
String serverUrl = configurationService.getProperty("oai.solr.url");
try {
server = new HttpSolrClient.Builder(serverUrl).build();
server = new HttpSolrClient.Builder(serverUrl)
.withHttpClient(httpConnectionPoolService.getClient())
.build();
log.debug("OAI Solr Server Initialized");
} catch (Exception e) {
log.error("Could not initialize OAI Solr Server at " + serverUrl , e);

10
dspace-oai/src/main/java/org/dspace/xoai/solr/DSpaceSolrServer.java
View File

@@ -13,6 +13,7 @@ import org.apache.logging.log4j.Logger;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.dspace.service.impl.HttpConnectionPoolService;
import org.dspace.services.ConfigurationService;
import org.dspace.services.factory.DSpaceServicesFactory;
@@ -33,9 +34,16 @@ public class DSpaceSolrServer {
if (_server == null) {
ConfigurationService configurationService
= DSpaceServicesFactory.getInstance().getConfigurationService();
HttpConnectionPoolService httpConnectionPoolService
= DSpaceServicesFactory.getInstance()
.getServiceManager()
.getServiceByName("solrHttpConnectionPoolService",
HttpConnectionPoolService.class);
String serverUrl = configurationService.getProperty("oai.solr.url");
try {
_server = new HttpSolrClient.Builder(serverUrl).build();
_server = new HttpSolrClient.Builder(serverUrl)
.withHttpClient(httpConnectionPoolService.getClient())
.build();
log.debug("OAI Solr Server Initialized");
} catch (Exception e) {
log.error("Could not initialize OAI Solr Server at " + serverUrl , e);

16
dspace/config/dspace.cfg
View File

@@ -45,6 +45,22 @@ default.language = en_US
# Since DSpace 7, SOLR must be installed as a stand-alone service
solr.server = http://localhost:8983/solr
# Solr connection pool.
# If you change these values, the changes are not effective until DSpace is
# restarted.
#
# Maximum open connections to Solr:
# solr.client.maxTotalConnections = 20
#
# Maximum open connections per Solr instance:
# solr.client.maxPerRoute = 15
#
# Default keep-alive time for open Solr connections, in milliseconds:
# solr.client.keepAlive = 5000
#
# Maximum lifetime of a pooled connection, in seconds:
# solr.client.timeToLive = 600
##### Database settings #####
# DSpace only supports two database types: PostgreSQL or Oracle

7
dspace/config/spring/api/core-services.xml
View File

@@ -63,6 +63,13 @@
<bean class="org.dspace.content.authority.ChoiceAuthorityServiceImpl"/>
<bean class="org.dspace.content.authority.MetadataAuthorityServiceImpl" lazy-init="true"/>
<bean class='org.dspace.service.impl.HttpConnectionPoolService'
id='solrHttpConnectionPoolService'
scope='singleton'
autowire-candidate='true'>
<constructor-arg name='configPrefix' value='solr'/>
</bean>
<!-- Ensure PluginService is initialized properly via init() method -->
<bean class="org.dspace.core.LegacyPluginServiceImpl" init-method="init"/>
<bean class="org.dspace.core.LicenseServiceImpl"/>