hazza/DSpace
1
0
Fork 0
mirror of https://github.com/DSpace/DSpace.git synced 2025-10-07 01:54:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'DS-3695' into mwoodiupui-DS-3695

Browse Source
This commit is contained in:
Mark H. Wood
2019-02-13 11:39:03 -05:00
committed by GitHub
parent cb3276b02e 69fcb0ebce
commit c1800427bd
8 changed files with 60 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
dspace-api/src/main/java/org/dspace/statistics/SolrLoggerServiceImpl.java
View File

@@ -971,6 +971,15 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea
.setFacetMinCount(1);
addAdditionalSolrYearCores(solrQuery);
// Can no longer set default field in schema
if (null == solrQuery.get("df")) {
solrQuery.add("df", "id");
}
// Can no longer set default match operator in schema
if (null == solrQuery.get("q.op")) {
solrQuery.add("q.op", "AND");
}
// Set the date facet if present
if (dateType != null) {
solrQuery.setParam("facet.date", "time")

33
dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java
View File

@@ -52,15 +52,15 @@ import org.dspace.content.MetadataField;
import org.dspace.content.MetadataValue;
import org.dspace.content.factory.ContentServiceFactory;
import org.dspace.content.service.ItemService;
import org.dspace.core.ConfigurationManager;
import org.dspace.core.Constants;
import org.dspace.core.Context;
import org.dspace.services.ConfigurationService;
import org.dspace.services.factory.DSpaceServicesFactory;
import org.dspace.xoai.exceptions.CompilingException;
import org.dspace.xoai.services.api.CollectionsService;
import org.dspace.xoai.services.api.cache.XOAICacheService;
import org.dspace.xoai.services.api.cache.XOAIItemCacheService;
import org.dspace.xoai.services.api.cache.XOAILastCompilationCacheService;
import org.dspace.xoai.services.api.config.ConfigurationService;
import org.dspace.xoai.services.api.solr.SolrServerResolver;
import org.dspace.xoai.solr.DSpaceSolrSearch;
import org.dspace.xoai.solr.exceptions.DSpaceSolrException;
@@ -92,6 +92,8 @@ public class XOAI {
private final AuthorizeService authorizeService;
private final ItemService itemService;
private final static ConfigurationService configurationService = DSpaceServicesFactory
.getInstance().getConfigurationService();
private List<String> getFileFormats(Item item) {
List<String> formats = new ArrayList<>();
@@ -281,14 +283,16 @@ public class XOAI {
throws DSpaceSolrIndexerException {
try {
int i = 0;
int batchSize = configurationService.getIntProperty("oai.import.batch.size", 1000);
SolrClient server = solrServerResolver.getServer();
ArrayList<SolrInputDocument> list = new ArrayList<>();
while (iterator.hasNext()) {
try {
Item item = iterator.next();
if (item.getHandle() == null) {
log.warn("Skipped item without handle: " + item.getID());
} else {
server.add(this.index(item));
list.add(this.index(item));
}
//Uncache the item to keep memory consumption low
context.uncacheEntity(item);
@@ -297,12 +301,20 @@ public class XOAI {
log.error(ex.getMessage(), ex);
}
i++;
if (i % 100 == 0) {
if (i % 1000 == 0 && batchSize != 1000) {
System.out.println(i + " items imported so far...");
}
if (i % batchSize == 0) {
System.out.println(i + " items imported so far...");
server.add(list);
server.commit();
list.clear();
}
}
System.out.println("Total: " + i + " items");
server.commit();
server.add(list);
server.commit(true, true);
list.clear();
return i;
} catch (SolrServerException | IOException ex) {
throw new DSpaceSolrIndexerException(ex.getMessage(), ex);
@@ -331,6 +343,7 @@ public class XOAI {
dates.add(policy.getEndDate());
}
}
context.uncacheEntity(policy);
}
dates.add(item.getLastModified());
Collections.sort(dates);
@@ -455,6 +468,7 @@ public class XOAI {
return true;
}
}
context.uncacheEntity(policy);
}
return false;
}
@@ -474,8 +488,8 @@ public class XOAI {
private static boolean getKnownExplanation(Throwable t) {
if (t instanceof ConnectException) {
System.err.println("Solr server ("
+ ConfigurationManager.getProperty("oai", "solr.url")
+ ") is down, turn it on.");
+ configurationService.getProperty("oai.solr.url", "")
+ ") is down, turn it on.");
return true;
}
@@ -522,7 +536,6 @@ public class XOAI {
BasicConfiguration.class
});
ConfigurationService configurationService = applicationContext.getBean(ConfigurationService.class);
XOAICacheService cacheService = applicationContext.getBean(XOAICacheService.class);
XOAIItemCacheService itemCacheService = applicationContext.getBean(XOAIItemCacheService.class);
@@ -544,7 +557,7 @@ public class XOAI {
boolean solr = true; // Assuming solr by default
solr = !("database").equals(configurationService.getProperty("oai", "storage"));
solr = !("database").equals(configurationService.getProperty("oai.storage", "solr"));
boolean run = false;
@@ -649,7 +662,7 @@ public class XOAI {
private static void usage() {
boolean solr = true; // Assuming solr by default
solr = !("database").equals(ConfigurationManager.getProperty("oai", "storage"));
solr = !("database").equals(configurationService.getProperty("oai.storage","solr"));
if (solr) {
System.out.println("OAI Manager Script");

10
dspace-oai/src/main/java/org/dspace/xoai/solr/DSpaceSolrSearch.java
View File

@@ -32,8 +32,16 @@ public class DSpaceSolrSearch {
public static SolrDocumentList query(SolrClient server, SolrQuery solrParams)
throws DSpaceSolrException, IOException {
solrParams.addSort("item.id", ORDER.asc);
// No longer can set default search field in the schema
if (null == solrParams.get("df")) {
solrParams.set("df", "item.handle");
}
// No longer can set default match operator in the schema
if (null == solrParams.get("q.op")) {
solrParams.set("q.op", "OR");
}
try {
solrParams.addSort("item.id", ORDER.asc);
QueryResponse response = server.query(solrParams);
return response.getResults();
} catch (SolrServerException ex) {

7
dspace/config/modules/oai.cfg
View File

@@ -31,6 +31,13 @@ oai.cache.enabled = true
# Base Cache Directory
oai.cache.dir = ${dspace.dir}/var/oai
#---------------------------------------------------------------#
#--------------OAI IMPORT CONFIGURATION ------------------------#
#---------------------------------------------------------------#
# Size of batches to commit to solr at a time
oai.import.batch.size = 1000
#---------------------------------------------------------------#
#--------------OAI HARVESTING CONFIGURATIONS--------------------#
#---------------------------------------------------------------#

3
dspace/solr/authority/conf/schema.xml
View File

@@ -20,6 +20,9 @@
This is the DSpace "authority" core, which holds records of attributions
provided by some metadata authority.
See dspace-api:org.dspace.authority.AuthoritySolrServiceImpl.
You should find the extensively commented example schema distributed with
Solr in [Solr]/server/solr/configsets/_default/conf/.
-->
<schema name="authority" version="1.1">

3
dspace/solr/oai/conf/schema.xml
View File

@@ -2,6 +2,9 @@
<!--
This is the DSpace "xoai" core.
See dspace-oai.
You should find the extensively commented example schema distributed with
Solr in [Solr]/server/solr/configsets/_default/conf/.
-->
<schema name="xoai" version="1.2">

3
dspace/solr/search/conf/schema.xml
View File

@@ -20,6 +20,9 @@
This is the DSpace "search" core, which supports full-text searching for
DSpace content bitstreams, Discovery, autocompletion, etc.
See dspace-api:org.dspace.discovery.SolrServiceImpl.
You should find the extensively commented example schema distributed with
Solr in [Solr]/server/solr/configsets/_default/conf/.
-->
<schema name="discovery" version="1.5">

3
dspace/solr/statistics/conf/schema.xml
View File

@@ -20,6 +20,9 @@
This is the DSpace "statistics" core, which holds records of each reference
to DSpace content. Each search or view/download event is logged here.
See dspace-api:org.dspace.statistics.SolrLoggerServiceImpl.
You should find the extensively commented example schema distributed with
Solr in [Solr]/server/solr/configsets/_default/conf/.
-->
<schema name="statistics" version="1.1">