hazza/DSpace
1
0
Fork 0
mirror of https://github.com/DSpace/DSpace.git synced 2025-10-14 05:23:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

[DS-3695] Ask Solr what cores it has, instead of assuming that we know where they are in the filesystem.

Browse Source 
(Solr might not even be installed on the same host.)
This commit is contained in:
Mark H. Wood
2019-08-15 10:16:10 -04:00
parent 4dfa7a370a
commit 8878b3a313
1 changed files with 68 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

108
dspace-api/src/main/java/org/dspace/statistics/SolrLoggerServiceImpl.java
View File

@@ -8,7 +8,6 @@
package org.dspace.statistics;
import java.io.File;
import java.io.FileFilter;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
@@ -17,7 +16,10 @@ import java.io.InputStreamReader;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.net.InetAddress;
import java.net.URI;
import java.net.URLEncoder;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.SQLException;
import java.text.DateFormat;
import java.text.ParseException;
@@ -53,6 +55,7 @@ import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.apache.solr.client.solrj.request.LukeRequest;
import org.apache.solr.client.solrj.response.CoreAdminResponse;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.LukeResponse;
import org.apache.solr.client.solrj.response.QueryResponse;
@@ -64,10 +67,12 @@ import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.luke.FieldFlag;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.util.NamedList;
import org.dspace.content.Bitstream;
import org.dspace.content.Bundle;
import org.dspace.content.Collection;
@@ -129,6 +134,12 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea
@Autowired(required = true)
private ClientInfoService clientInfoService;
/** URL to the current-year statistics core. Prior-year shards will have a year suffixed. */
private String statisticsCoreURL;
/** Name of the current-year statistics core. Prior-year shards will have a year suffixed. */
private String statisticsCoreBase;
public static enum StatisticsType {
VIEW("view"),
SEARCH("search"),
@@ -153,14 +164,26 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea
@Override
public void afterPropertiesSet() throws Exception {
log.info("solr-statistics.server:" + configurationService.getProperty("solr-statistics.server"));
log.info("usage-statistics.dbfile:" + configurationService.getProperty("usage-statistics.dbfile"));
statisticsCoreURL = configurationService.getProperty("solr-statistics.server");
if (null != statisticsCoreURL) {
Path statisticsPath = Paths.get(new URI(statisticsCoreURL));
statisticsCoreBase = statisticsPath
.getName(statisticsPath.getNameCount() - 1)
.toString();
} else {
statisticsCoreBase = null;
}
log.info("solr-statistics.server: {}", statisticsCoreURL);
log.info("usage-statistics.dbfile: {}",
configurationService.getProperty("usage-statistics.dbfile"));
HttpSolrClient server = null;
if (configurationService.getProperty("solr-statistics.server") != null) {
if (statisticsCoreURL != null) {
try {
server = new HttpSolrClient.Builder(configurationService.getProperty("solr-statistics.server")).build();
server = new HttpSolrClient.Builder(statisticsCoreURL).build();
} catch (Exception e) {
log.error("Error accessing Solr server configured in 'solr-statistics.server'", e);
}
@@ -1159,7 +1182,7 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea
filterQuery.append(")");
Map<String, String> yearQueryParams = new HashMap<String, String>();
Map<String, String> yearQueryParams = new HashMap<>();
yearQueryParams.put(CommonParams.Q, "*:*");
yearQueryParams.put(CommonParams.ROWS, String.valueOf(10000));
yearQueryParams.put(CommonParams.FQ, filterQuery.toString());
@@ -1170,7 +1193,7 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea
yearQueryParams.put("csv.mv.separator", MULTIPLE_VALUES_SPLITTER);
//Start by creating a new core
String coreName = "statistics-" + dcStart.getYearUTC();
String coreName = statisticsCoreBase + "-" + dcStart.getYearUTC();
HttpSolrClient statisticsYearServer = createCore((HttpSolrClient) solr, coreName);
System.out.println("Moving: " + totalRecords + " into core " + coreName);
@@ -1229,20 +1252,22 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea
FileUtils.deleteDirectory(tempDirectory);
}
protected HttpSolrClient createCore(HttpSolrClient solr, String coreName) throws IOException, SolrServerException {
String solrDir = configurationService.getProperty("dspace.dir") + File.separator + "solr" + File.separator;
String baseSolrUrl = solr.getBaseURL().replace("statistics", "");
protected HttpSolrClient createCore(HttpSolrClient solr, String coreName)
throws IOException, SolrServerException {
String baseSolrUrl = solr.getBaseURL().replace(statisticsCoreBase, "");
//DS-3458: Test to see if a solr core already exists. If it exists, return that server. Otherwise create a
// new one.
//DS-3458: Test to see if a solr core already exists. If it exists,
// return a connection to that core. Otherwise create a new core and
// return a connection to it.
HttpSolrClient returnServer = new HttpSolrClient.Builder(baseSolrUrl + "/" + coreName).build();
try {
SolrPingResponse ping = returnServer.ping();
log.debug(String.format("Ping of Solr Core [%s] Returned with Status [%d]", coreName, ping.getStatus()));
log.debug("Ping of Solr Core {} returned with Status {}",
coreName, ping.getStatus());
return returnServer;
} catch (Exception e) {
log.debug(String.format("Ping of Solr Core [%s] Failed with [%s]. New Core Will be Created", coreName,
e.getClass().getName()));
} catch (IOException | SolrServerException e) {
log.debug("Ping of Solr Core {} failed with {}. New Core Will be Created",
coreName, e.getClass().getName());
}
//Unfortunately, this class is documented as "experimental and subject to change" on the Lucene website.
@@ -1250,9 +1275,6 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea
CoreAdminRequest.Create create = new CoreAdminRequest.Create();
create.setCoreName(coreName);
//The config files for a statistics shard reside wihtin the statistics repository
create.setInstanceDir("statistics");
create.setDataDir(solrDir + coreName + File.separator + "data");
HttpSolrClient solrServer = new HttpSolrClient.Builder(baseSolrUrl).build();
create.process(solrServer);
log.info("Created core with name: " + coreName);
@@ -1520,43 +1542,49 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea
}
/*
* The statistics shards should not be initialized until all tomcat webapps are fully initialized.
* DS-3457 uncovered an issue in DSpace 6x in which this code triggered tomcat to hang when statistics shards are
* present.
* This code is synchonized in the event that 2 threads trigger the initialization at the same time.
* The statistics shards should not be initialized until all tomcat webapps
* are fully initialized. DS-3457 uncovered an issue in DSpace 6x in which
* this code triggered Tomcat to hang when statistics shards are present.
* This code is synchonized in the event that 2 threads trigger the
* initialization at the same time.
*/
protected synchronized void initSolrYearCores() {
if (statisticYearCoresInit || !(solr instanceof HttpSolrClient)) {
return;
}
try {
//Base url should like : http://localhost:{port.number}/solr
String baseSolrUrl = ((HttpSolrClient) solr).getBaseURL().replace(statisticsCoreBase, "");
try (HttpSolrClient enumClient = new HttpSolrClient.Builder(baseSolrUrl).build();) {
//Attempt to retrieve all the statistic year cores
File solrDir = new File(
configurationService.getProperty("dspace.dir") + File.separator + "solr" + File.separator);
File[] solrCoreFiles = solrDir.listFiles(new FileFilter() {
@Override
public boolean accept(File file) {
//Core name example: statistics-2008
return file.getName().matches("statistics-\\d\\d\\d\\d");
CoreAdminRequest coresRequest = new CoreAdminRequest();
coresRequest.setAction(CoreAdminAction.STATUS);
CoreAdminResponse coresResponse = coresRequest.process(enumClient);
NamedList<Object> response = coresResponse.getResponse();
NamedList<Object> coreStatuses = (NamedList<Object>) response.get("status");
List<String> statCoreNames = new ArrayList<>(coreStatuses.size());
for (Map.Entry<String, Object> coreStatus : coreStatuses) {
String coreName = coreStatus.getKey();
if (coreName.startsWith(statisticsCoreBase)) {
statCoreNames.add(coreName);
}
});
//Base url should like : http://localhost:{port.number}/solr
String baseSolrUrl = ((HttpSolrClient) solr).getBaseURL().replace("statistics", "");
for (File solrCoreFile : solrCoreFiles) {
log.info("Loading core with name: " + solrCoreFile.getName());
}
createCore((HttpSolrClient) solr, solrCoreFile.getName());
for (String statCoreName : statCoreNames) {
log.info("Loading core with name: " + statCoreName);
createCore((HttpSolrClient) solr, statCoreName);
//Add it to our cores list so we can query it !
statisticYearCores
.add(baseSolrUrl.replace("http://", "").replace("https://", "") + solrCoreFile.getName());
.add(baseSolrUrl.replace("http://", "").replace("https://", "") + statCoreName);
}
//Also add the core containing the current year !
statisticYearCores.add(((HttpSolrClient) solr)
.getBaseURL()
.replace("http://", "")
.replace("https://", ""));
} catch (Exception e) {
} catch (IOException | SolrServerException e) {
log.error(e.getMessage(), e);
}
statisticYearCoresInit = true;