diff --git a/dspace-api/pom.xml b/dspace-api/pom.xml index 60fcc44060..a5bfd61bfb 100644 --- a/dspace-api/pom.xml +++ b/dspace-api/pom.xml @@ -759,6 +759,13 @@ 20180130 + + + com.opencsv + opencsv + 4.5 + + org.apache.velocity velocity-engine-core diff --git a/dspace-api/src/main/java/org/dspace/authority/AuthorityValue.java b/dspace-api/src/main/java/org/dspace/authority/AuthorityValue.java index acb5bd754d..10a608bb76 100644 --- a/dspace-api/src/main/java/org/dspace/authority/AuthorityValue.java +++ b/dspace-api/src/main/java/org/dspace/authority/AuthorityValue.java @@ -8,6 +8,7 @@ package org.dspace.authority; import java.sql.SQLException; +import java.text.DateFormat; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -23,6 +24,7 @@ import org.dspace.content.Item; import org.dspace.content.MetadataValue; import org.dspace.content.factory.ContentServiceFactory; import org.dspace.core.Context; +import org.dspace.util.SolrUtils; import org.joda.time.DateTime; import org.joda.time.format.DateTimeFormatter; import org.joda.time.format.ISODateTimeFormat; @@ -34,8 +36,6 @@ import org.joda.time.format.ISODateTimeFormat; * @author Mark Diggory (markd at atmire dot com) */ public class AuthorityValue { - - /** * The id of the record in solr */ @@ -150,12 +150,13 @@ public class AuthorityValue { public SolrInputDocument getSolrInputDocument() { SolrInputDocument doc = new SolrInputDocument(); + DateFormat solrDateFormatter = SolrUtils.getDateFormatter(); doc.addField("id", getId()); doc.addField("field", getField()); doc.addField("value", getValue()); doc.addField("deleted", isDeleted()); - doc.addField("creation_date", getCreationDate()); - doc.addField("last_modified_date", getLastModified()); + doc.addField("creation_date", solrDateFormatter.format(getCreationDate())); + doc.addField("last_modified_date", solrDateFormatter.format(getLastModified())); doc.addField("authority_type", getAuthorityType()); return doc; } @@ -196,12 +197,12 @@ public class AuthorityValue { * @return map */ public Map choiceSelectMap() { - return new HashMap(); + return new HashMap<>(); } public List getDateFormatters() { - List list = new ArrayList(); + List list = new ArrayList<>(); list.add(ISODateTimeFormat.dateTime()); list.add(ISODateTimeFormat.dateTimeNoMillis()); return list; diff --git a/dspace-api/src/main/java/org/dspace/discovery/SolrServiceImpl.java b/dspace-api/src/main/java/org/dspace/discovery/SolrServiceImpl.java index acb259f99d..6de698d0ed 100644 --- a/dspace-api/src/main/java/org/dspace/discovery/SolrServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/discovery/SolrServiceImpl.java @@ -14,6 +14,7 @@ import java.io.PrintWriter; import java.io.Serializable; import java.io.StringWriter; import java.sql.SQLException; +import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -105,6 +106,7 @@ import org.dspace.handle.service.HandleService; import org.dspace.services.factory.DSpaceServicesFactory; import org.dspace.storage.rdbms.DatabaseUtils; import org.dspace.util.MultiFormatDateParser; +import org.dspace.util.SolrUtils; import org.dspace.workflow.WorkflowItem; import org.dspace.xmlworkflow.WorkflowConfigurationException; import org.dspace.xmlworkflow.factory.XmlWorkflowFactory; @@ -1038,6 +1040,8 @@ public class SolrServiceImpl implements SearchService, IndexingService { */ protected void buildDocument(Context context, Item item) throws SQLException, IOException { + final DateFormat solrDateFormatter = SolrUtils.getDateFormatter(); + String handle = item.getHandle(); if (handle == null) { @@ -1055,7 +1059,7 @@ public class SolrServiceImpl implements SearchService, IndexingService { doc.addField("archived", item.isArchived()); doc.addField("withdrawn", item.isWithdrawn()); doc.addField("discoverable", item.isDiscoverable()); - doc.addField("lastModified", item.getLastModified()); + doc.addField("lastModified", solrDateFormatter.format(item.getLastModified())); EPerson submitter = item.getSubmitter(); if (submitter != null) { @@ -1476,7 +1480,9 @@ public class SolrServiceImpl implements SearchService, IndexingService { if (type.equals(DiscoveryConfigurationParameters.TYPE_DATE)) { Date date = MultiFormatDateParser.parse(value); if (date != null) { - doc.addField(field + "_dt", date); + final DateFormat solrDateFormatter = SolrUtils.getDateFormatter(); + String stringDate = solrDateFormatter.format(date); + doc.addField(field + "_dt", stringDate); } else { log.warn("Error while indexing sort date field, item: " + item .getHandle() + " metadata field: " + field + " date value: " + date); @@ -1741,7 +1747,8 @@ public class SolrServiceImpl implements SearchService, IndexingService { List locations) { // want to be able to check when last updated // (not tokenized, but it is indexed) - doc.addField(LAST_INDEXED_FIELD, new Date()); + doc.addField(LAST_INDEXED_FIELD, + SolrUtils.getDateFormatter().format(new Date())); // New fields to weaken the dependence on handles, and allow for faster // list display diff --git a/dspace-api/src/main/java/org/dspace/statistics/SolrLoggerServiceImpl.java b/dspace-api/src/main/java/org/dspace/statistics/SolrLoggerServiceImpl.java index f34dca7d20..e1ff0c69b8 100644 --- a/dspace-api/src/main/java/org/dspace/statistics/SolrLoggerServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/statistics/SolrLoggerServiceImpl.java @@ -8,7 +8,6 @@ package org.dspace.statistics; import java.io.File; -import java.io.FileFilter; import java.io.FileNotFoundException; import java.io.FileWriter; import java.io.IOException; @@ -17,7 +16,10 @@ import java.io.InputStreamReader; import java.io.Reader; import java.io.UnsupportedEncodingException; import java.net.InetAddress; +import java.net.URI; import java.net.URLEncoder; +import java.nio.file.Path; +import java.nio.file.Paths; import java.sql.SQLException; import java.text.DateFormat; import java.text.ParseException; @@ -45,14 +47,18 @@ import org.apache.commons.lang3.time.DateFormatUtils; import org.apache.http.HttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient.RemoteSolrException; import org.apache.solr.client.solrj.request.AbstractUpdateRequest; import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest; import org.apache.solr.client.solrj.request.CoreAdminRequest; import org.apache.solr.client.solrj.request.LukeRequest; +import org.apache.solr.client.solrj.response.CoreAdminResponse; import org.apache.solr.client.solrj.response.FacetField; import org.apache.solr.client.solrj.response.LukeResponse; import org.apache.solr.client.solrj.response.QueryResponse; @@ -64,10 +70,12 @@ import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.luke.FieldFlag; import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction; import org.apache.solr.common.params.FacetParams; import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.ShardParams; +import org.apache.solr.common.util.NamedList; import org.dspace.content.Bitstream; import org.dspace.content.Bundle; import org.dspace.content.Collection; @@ -103,8 +111,7 @@ import org.springframework.beans.factory.annotation.Autowired; */ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBean { - private static final org.apache.logging.log4j.Logger log = - org.apache.logging.log4j.LogManager.getLogger(SolrLoggerServiceImpl.class); + private static final Logger log = LogManager.getLogger(); private static final String MULTIPLE_VALUES_SPLITTER = "|"; protected SolrClient solr; @@ -129,6 +136,12 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea @Autowired(required = true) private ClientInfoService clientInfoService; + /** URL to the current-year statistics core. Prior-year shards will have a year suffixed. */ + private String statisticsCoreURL; + + /** Name of the current-year statistics core. Prior-year shards will have a year suffixed. */ + private String statisticsCoreBase; + public static enum StatisticsType { VIEW("view"), SEARCH("search"), @@ -153,14 +166,26 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea @Override public void afterPropertiesSet() throws Exception { - log.info("solr-statistics.server:" + configurationService.getProperty("solr-statistics.server")); - log.info("usage-statistics.dbfile:" + configurationService.getProperty("usage-statistics.dbfile")); + statisticsCoreURL = configurationService.getProperty("solr-statistics.server"); + + if (null != statisticsCoreURL) { + Path statisticsPath = Paths.get(new URI(statisticsCoreURL).getPath()); + statisticsCoreBase = statisticsPath + .getName(statisticsPath.getNameCount() - 1) + .toString(); + } else { + statisticsCoreBase = null; + } + + log.info("solr-statistics.server: {}", statisticsCoreURL); + log.info("usage-statistics.dbfile: {}", + configurationService.getProperty("usage-statistics.dbfile")); HttpSolrClient server = null; - if (configurationService.getProperty("solr-statistics.server") != null) { + if (statisticsCoreURL != null) { try { - server = new HttpSolrClient.Builder(configurationService.getProperty("solr-statistics.server")).build(); + server = new HttpSolrClient.Builder(statisticsCoreURL).build(); } catch (Exception e) { log.error("Error accessing Solr server configured in 'solr-statistics.server'", e); } @@ -1159,7 +1184,7 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea filterQuery.append(")"); - Map yearQueryParams = new HashMap(); + Map yearQueryParams = new HashMap<>(); yearQueryParams.put(CommonParams.Q, "*:*"); yearQueryParams.put(CommonParams.ROWS, String.valueOf(10000)); yearQueryParams.put(CommonParams.FQ, filterQuery.toString()); @@ -1170,7 +1195,7 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea yearQueryParams.put("csv.mv.separator", MULTIPLE_VALUES_SPLITTER); //Start by creating a new core - String coreName = "statistics-" + dcStart.getYearUTC(); + String coreName = statisticsCoreBase + "-" + dcStart.getYearUTC(); HttpSolrClient statisticsYearServer = createCore((HttpSolrClient) solr, coreName); System.out.println("Moving: " + totalRecords + " into core " + coreName); @@ -1198,12 +1223,12 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea for (File tempCsv : filesToUpload) { //Upload the data in the csv files to our new solr core - ContentStreamUpdateRequest contentStreamUpdateRequest = new ContentStreamUpdateRequest("/update/csv"); - contentStreamUpdateRequest.setParam("stream.contentType", "text/plain;charset=utf-8"); + ContentStreamUpdateRequest contentStreamUpdateRequest = new ContentStreamUpdateRequest("/update"); + contentStreamUpdateRequest.setParam("stream.contentType", "text/csv;charset=utf-8"); contentStreamUpdateRequest.setParam("escape", "\\"); contentStreamUpdateRequest.setParam("skip", "_version_"); contentStreamUpdateRequest.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true); - contentStreamUpdateRequest.addFile(tempCsv, "text/plain;charset=utf-8"); + contentStreamUpdateRequest.addFile(tempCsv, "text/csv;charset=utf-8"); //Add parsing directives for the multivalued fields so that they are stored as separate values // instead of one value @@ -1223,39 +1248,42 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea solr.deleteByQuery(filterQuery.toString()); solr.commit(true, true); - log.info("Moved " + totalRecords + " records into core: " + coreName); + log.info("Moved {} records into core: {}", totalRecords, coreName); } FileUtils.deleteDirectory(tempDirectory); } - protected HttpSolrClient createCore(HttpSolrClient solr, String coreName) throws IOException, SolrServerException { - String solrDir = configurationService.getProperty("dspace.dir") + File.separator + "solr" + File.separator; - String baseSolrUrl = solr.getBaseURL().replace("statistics", ""); + protected HttpSolrClient createCore(HttpSolrClient solr, String coreName) + throws IOException, SolrServerException { + String baseSolrUrl = solr.getBaseURL().replace(statisticsCoreBase, ""); // Has trailing slash - //DS-3458: Test to see if a solr core already exists. If it exists, return that server. Otherwise create a - // new one. - HttpSolrClient returnServer = new HttpSolrClient.Builder(baseSolrUrl + "/" + coreName).build(); + //DS-3458: Test to see if a solr core already exists. If it exists, + // return a connection to that core. Otherwise create a new core and + // return a connection to it. + HttpSolrClient returnServer = new HttpSolrClient.Builder(baseSolrUrl + coreName).build(); try { SolrPingResponse ping = returnServer.ping(); - log.debug(String.format("Ping of Solr Core [%s] Returned with Status [%d]", coreName, ping.getStatus())); + log.debug("Ping of Solr Core {} returned with Status {}", + coreName, ping.getStatus()); return returnServer; - } catch (Exception e) { - log.debug(String.format("Ping of Solr Core [%s] Failed with [%s]. New Core Will be Created", coreName, - e.getClass().getName())); + } catch (IOException | RemoteSolrException | SolrServerException e) { + log.debug("Ping of Solr Core {} failed with {}. New Core Will be Created", + coreName, e.getClass().getName()); } //Unfortunately, this class is documented as "experimental and subject to change" on the Lucene website. //http://lucene.apache.org/solr/4_4_0/solr-solrj/org/apache/solr/client/solrj/request/CoreAdminRequest.html CoreAdminRequest.Create create = new CoreAdminRequest.Create(); create.setCoreName(coreName); + String configSetName = configurationService + .getProperty("solr-statistics.configset", "statistics"); + create.setConfigSet(configSetName); + create.setInstanceDir(coreName); - //The config files for a statistics shard reside wihtin the statistics repository - create.setInstanceDir("statistics"); - create.setDataDir(solrDir + coreName + File.separator + "data"); HttpSolrClient solrServer = new HttpSolrClient.Builder(baseSolrUrl).build(); create.process(solrServer); - log.info("Created core with name: " + coreName); + log.info("Created core with name: {} from configset {}", coreName, configSetName); return returnServer; } @@ -1390,10 +1418,10 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea //Add all the separate csv files for (File tempCsv : tempCsvFiles) { - ContentStreamUpdateRequest contentStreamUpdateRequest = new ContentStreamUpdateRequest("/update/csv"); - contentStreamUpdateRequest.setParam("stream.contentType", "text/plain;charset=utf-8"); + ContentStreamUpdateRequest contentStreamUpdateRequest = new ContentStreamUpdateRequest("/update"); + contentStreamUpdateRequest.setParam("stream.contentType", "text/csv;charset=utf-8"); contentStreamUpdateRequest.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true); - contentStreamUpdateRequest.addFile(tempCsv, "text/plain;charset=utf-8"); + contentStreamUpdateRequest.addFile(tempCsv, "text/csv;charset=utf-8"); solr.request(contentStreamUpdateRequest); } @@ -1520,43 +1548,49 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea } /* - * The statistics shards should not be initialized until all tomcat webapps are fully initialized. - * DS-3457 uncovered an issue in DSpace 6x in which this code triggered tomcat to hang when statistics shards are - * present. - * This code is synchonized in the event that 2 threads trigger the initialization at the same time. + * The statistics shards should not be initialized until all tomcat webapps + * are fully initialized. DS-3457 uncovered an issue in DSpace 6x in which + * this code triggered Tomcat to hang when statistics shards are present. + * This code is synchonized in the event that 2 threads trigger the + * initialization at the same time. */ protected synchronized void initSolrYearCores() { if (statisticYearCoresInit || !(solr instanceof HttpSolrClient)) { return; } - try { + + //Base url should like : http://localhost:{port.number}/solr + String baseSolrUrl = ((HttpSolrClient) solr).getBaseURL().replace(statisticsCoreBase, ""); + + try (HttpSolrClient enumClient = new HttpSolrClient.Builder(baseSolrUrl).build();) { //Attempt to retrieve all the statistic year cores - File solrDir = new File( - configurationService.getProperty("dspace.dir") + File.separator + "solr" + File.separator); - File[] solrCoreFiles = solrDir.listFiles(new FileFilter() { - - @Override - public boolean accept(File file) { - //Core name example: statistics-2008 - return file.getName().matches("statistics-\\d\\d\\d\\d"); + CoreAdminRequest coresRequest = new CoreAdminRequest(); + coresRequest.setAction(CoreAdminAction.STATUS); + CoreAdminResponse coresResponse = coresRequest.process(enumClient); + NamedList response = coresResponse.getResponse(); + NamedList coreStatuses = (NamedList) response.get("status"); + List statCoreNames = new ArrayList<>(coreStatuses.size()); + for (Map.Entry coreStatus : coreStatuses) { + String coreName = coreStatus.getKey(); + if (coreName.startsWith(statisticsCoreBase)) { + statCoreNames.add(coreName); } - }); - //Base url should like : http://localhost:{port.number}/solr - String baseSolrUrl = ((HttpSolrClient) solr).getBaseURL().replace("statistics", ""); - for (File solrCoreFile : solrCoreFiles) { - log.info("Loading core with name: " + solrCoreFile.getName()); + } - createCore((HttpSolrClient) solr, solrCoreFile.getName()); + for (String statCoreName : statCoreNames) { + log.info("Loading core with name: " + statCoreName); + + createCore((HttpSolrClient) solr, statCoreName); //Add it to our cores list so we can query it ! statisticYearCores - .add(baseSolrUrl.replace("http://", "").replace("https://", "") + solrCoreFile.getName()); + .add(baseSolrUrl.replace("http://", "").replace("https://", "") + statCoreName); } //Also add the core containing the current year ! statisticYearCores.add(((HttpSolrClient) solr) .getBaseURL() .replace("http://", "") .replace("https://", "")); - } catch (Exception e) { + } catch (IOException | SolrServerException e) { log.error(e.getMessage(), e); } statisticYearCoresInit = true; diff --git a/dspace-api/src/main/java/org/dspace/statistics/package-info.java b/dspace-api/src/main/java/org/dspace/statistics/package-info.java new file mode 100644 index 0000000000..452a519bf9 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/statistics/package-info.java @@ -0,0 +1,16 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +/** + * Facilities for detecting and storing both model and usage events, and for + * querying the store. + * + * See {@link SolrLoggerUsageEventListener} and {@link StatisticsLoggingConsumer} + * for event capture, and {@link SolrLoggerServiceImpl} for storing and querying. + */ + +package org.dspace.statistics; diff --git a/dspace-api/src/main/java/org/dspace/statistics/util/RepairDump.java b/dspace-api/src/main/java/org/dspace/statistics/util/RepairDump.java new file mode 100644 index 0000000000..f56a3226ee --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/statistics/util/RepairDump.java @@ -0,0 +1,125 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.statistics.util; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.io.Reader; +import java.io.Writer; +import java.lang.invoke.MethodHandles; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.UUID; + +import com.opencsv.CSVReader; +import com.opencsv.CSVWriter; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.lang3.StringUtils; + +/** + * Repair various problems in a statistics core export CSV. + * @author Mark H. Wood + */ +public class RepairDump { + private RepairDump() {} + + /** + * Repair known classes of problems with exported statistics. + * Reads standard input, writes repaired CSV to standard output. + * @param args + */ + public static void main( String[] args ) { + long recordCount = 0; // Input record counter. + long repairCount = 0; // Repaired records counter. + + boolean verbose; // Give more information about what's happening. + + // Analyze the command line. + Options options = new Options(); + options.addOption("h", "help", false, "Give help on options."); + options.addOption("v", "verbose", false, "Write extra information to standard error."); + + CommandLine command = null; + try { + command = new DefaultParser().parse(options, args); + } catch (ParseException ex) { + System.err.println(ex.getMessage()); + System.exit(1); + } + + if (command.hasOption("h")) { + giveHelp(options); + System.exit(0); + } + + verbose = command.hasOption("v"); + + // Copy standard in to standard out, fixing problems. + try ( + Reader input = new InputStreamReader(System.in, StandardCharsets.UTF_8); + CSVReader csvReader = new CSVReader(input); + Writer output = new OutputStreamWriter(System.out, StandardCharsets.UTF_8); + CSVWriter csvWriter = new CSVWriter(output); + ) { + // Read the column headers. + String[] fields = csvReader.readNext(); + + // Which column is "uid"? + int uidIndex = -1; + for (int i = 0; i < fields.length; i++) { + if (fields[i].equals("uid")) { + uidIndex = i; + break; + } + } + if (uidIndex < 0) { + System.err.println("Error: input contains no 'uid' column."); + System.exit(1); + } + + // Copy the headers to output. + csvWriter.writeNext(fields); + + // Copy records to output, repairing any problems that we find. + while (null != (fields = csvReader.readNext())) { + recordCount++; + + // Set 'uid' to a new random UUID if empty. + if (fields.length < uidIndex + 1) { // Too short to have 'uid' + fields = Arrays.copyOf(fields, uidIndex); + } + if (StringUtils.isBlank(fields[uidIndex])) { // 'uid' field is empty. + if (verbose) { + System.err.format("Missing 'uid' at record %d%n", recordCount); + } + fields[uidIndex] = UUID.randomUUID().toString(); + repairCount++; + } + + // Write repaired record. + csvWriter.writeNext(fields); + } + } catch (IOException ex) { + System.err.format("Could not read the export at record %d: ", recordCount); + System.err.println(ex.getMessage()); + } finally { + System.err.format("Repaired %d out of %d records.%n", + repairCount, recordCount); + } + } + + private static void giveHelp(Options options) { + String className = MethodHandles.lookup().lookupClass().getCanonicalName(); + new HelpFormatter().printHelp(className + " [options]", options); + } +} diff --git a/dspace-api/src/main/java/org/dspace/util/SolrUtils.java b/dspace-api/src/main/java/org/dspace/util/SolrUtils.java new file mode 100644 index 0000000000..f62feba298 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/util/SolrUtils.java @@ -0,0 +1,40 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ + +package org.dspace.util; + +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.time.ZoneOffset; +import java.util.TimeZone; + +/** + * Common constants and static methods for working with Solr. + * + * @author Mark H. Wood + */ +public class SolrUtils { + /** Solr uses UTC always. */ + public static final TimeZone SOLR_TIME_ZONE = TimeZone.getTimeZone(ZoneOffset.UTC); + + /** Restricted ISO 8601 format used by Solr. */ + public static final String SOLR_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; + + /** Do not instantiate. */ + private SolrUtils() { } + + /** + * Create a formatter configured for Solr-style date strings and the UTC time zone. + * @see SOLR_DATE_FORMAT + * + * @return date formatter compatible with Solr. + */ + public static DateFormat getDateFormatter() { + return new SimpleDateFormat(SolrUtils.SOLR_DATE_FORMAT); + } +} diff --git a/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java b/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java index 0186798b1a..8704324a88 100644 --- a/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java +++ b/dspace-oai/src/main/java/org/dspace/xoai/app/XOAI.java @@ -56,6 +56,7 @@ import org.dspace.core.Constants; import org.dspace.core.Context; import org.dspace.services.ConfigurationService; import org.dspace.services.factory.DSpaceServicesFactory; +import org.dspace.util.SolrUtils; import org.dspace.xoai.exceptions.CompilingException; import org.dspace.xoai.services.api.CollectionsService; import org.dspace.xoai.services.api.cache.XOAICacheService; @@ -313,7 +314,9 @@ public class XOAI { } System.out.println("Total: " + i + " items"); if (i > 0) { - server.add(list); + if (!list.isEmpty()) { + server.add(list); + } server.commit(true, true); list.clear(); } @@ -413,7 +416,8 @@ public class XOAI { * relevant policy dates and the standard lastModified date and take the * most recent of those which have already passed. */ - doc.addField("item.lastmodified", this.getMostRecentModificationDate(item)); + doc.addField("item.lastmodified", SolrUtils.getDateFormatter() + .format(this.getMostRecentModificationDate(item))); if (item.getSubmitter() != null) { doc.addField("item.submitter", item.getSubmitter().getEmail()); diff --git a/dspace/config/modules/solr-statistics.cfg b/dspace/config/modules/solr-statistics.cfg index ebc65496d7..d73baea7cb 100644 --- a/dspace/config/modules/solr-statistics.cfg +++ b/dspace/config/modules/solr-statistics.cfg @@ -15,6 +15,10 @@ solr-statistics.server = ${solr.server}/statistics # A comma-separated list that contains the bundles for which the bitstreams will be displayed solr-statistics.query.filter.bundles=ORIGINAL +# Name of the "configset" (bundle of template core files) which will be used to +# create new Solr cores when sharding the statistics data. +solr-statistics.configset = statistics + # control solr statistics querying to filter out spider IPs # false by default #solr-statistics.query.filter.spiderIp = false @@ -30,4 +34,4 @@ solr-statistics.spiderips.urls = http://iplists.com/google.txt, \ http://iplists.com/infoseek.txt, \ http://iplists.com/altavista.txt, \ http://iplists.com/excite.txt, \ - http://iplists.com/misc.txt \ No newline at end of file + http://iplists.com/misc.txt diff --git a/dspace/solr/oai/conf/schema.xml b/dspace/solr/oai/conf/schema.xml index 84e92bfe96..b2f61c2de9 100644 --- a/dspace/solr/oai/conf/schema.xml +++ b/dspace/solr/oai/conf/schema.xml @@ -143,14 +143,17 @@ + - - + + + + diff --git a/dspace/solr/search/conf/schema.xml b/dspace/solr/search/conf/schema.xml index 1d98a94c39..f9ace7461a 100644 --- a/dspace/solr/search/conf/schema.xml +++ b/dspace/solr/search/conf/schema.xml @@ -321,7 +321,6 @@ - diff --git a/dspace/src/main/config/build.xml b/dspace/src/main/config/build.xml index 87b01533d8..98414f0cdd 100644 --- a/dspace/src/main/config/build.xml +++ b/dspace/src/main/config/build.xml @@ -82,10 +82,6 @@ Common usage: - - - - @@ -124,7 +120,6 @@ Common usage: - @@ -173,7 +168,9 @@ Common usage: - + @@ -371,7 +368,10 @@ Common usage: - + - - Checking if any Solr indexes (${dspace.dir}/solr/*) need upgrading... - - - - - - - - - - Current version of Solr/Lucene: ${latest_version} - - - - - - - - - - - - Cleanup any downloaded lucene-core-*.jar files. We don't need them anymore. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Checking if the Solr index at ${indexDir} is >= Solr ${version} - - - - - - - - - - - - - - -ERROR occurred while checking Solr index version: -${version_compare} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading Solr/Lucene Index at ${indexDir} to Solr/Lucene ${version}. - - - - - - - - - - - - - - Downloading ${lucene-core.jar} - - - - - - -==================================================================== -WARNING : FAILED TO DOWNLOAD LUCENE-CORE.JAR - (Needed to upgrade your existing Solr indexes) - -Underlying Error: ${lucene.download.error} - -In order to upgrade your Solr indexes to the latest version, -you will need to do the following: - -(1) Manually download the lucene-core-${version}.jar at: - - ${lucene-core.jar} - -(2) Place the JAR in the "[src]/dspace/target/dspace-installer" directory. - - [src]/dspace/target/dspace-installer/${lucene-core}.jar - -(3) Manually re-run the following 'ant' upgrade command: - - ant update_solr_indexes - -For more information, please see the Upgrade Instructions. -==================================================================== - - - - - - - - - - - - - - Upgrading the Solr index in ${indexDir}. Depending on the index size, this may take a while (please be patient)... - - - - - - - - - - - - - - Upgrading the Solr index in ${indexDir}. Depending on the index size, this may take a while (please be patient)... - - - - - - - - - -