diff --git a/dspace-api/pom.xml b/dspace-api/pom.xml index 6e2d340d41..f9ca96ffd6 100644 --- a/dspace-api/pom.xml +++ b/dspace-api/pom.xml @@ -347,6 +347,61 @@ jbibtex 1.0.0 + + org.apache.solr + solr-solrj + ${lucene.version} + + + org.slf4j + jcl-over-slf4j + + + org.slf4j + slf4j-api + + + + + commons-configuration + commons-configuration + 1.8 + + + org.dspace.dependencies + dspace-geoip + 1.2.3 + + + org.apache.ant + ant + + + org.dspace.dnsjava + dnsjava + 2.0.6 + + + + org.ostermiller + utils + 1.07.00 + + + + org.elasticsearch + elasticsearch + 0.18.6 + + + + + com.google.code.gson + gson + 2.1 + compile + + diff --git a/dspace-stats/src/main/java/org/dspace/statistics/DataTermsFacet.java b/dspace-api/src/main/java/org/dspace/statistics/DataTermsFacet.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/DataTermsFacet.java rename to dspace-api/src/main/java/org/dspace/statistics/DataTermsFacet.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/Dataset.java b/dspace-api/src/main/java/org/dspace/statistics/Dataset.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/Dataset.java rename to dspace-api/src/main/java/org/dspace/statistics/Dataset.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/ElasticSearchLogger.java b/dspace-api/src/main/java/org/dspace/statistics/ElasticSearchLogger.java similarity index 99% rename from dspace-stats/src/main/java/org/dspace/statistics/ElasticSearchLogger.java rename to dspace-api/src/main/java/org/dspace/statistics/ElasticSearchLogger.java index bfc3d623d8..70151960dd 100644 --- a/dspace-stats/src/main/java/org/dspace/statistics/ElasticSearchLogger.java +++ b/dspace-api/src/main/java/org/dspace/statistics/ElasticSearchLogger.java @@ -221,7 +221,7 @@ public class ElasticSearchLogger { "} } }"; client.prepareIndex(indexName, indexType, "1") - .setSource(jsonBuilder() + .setSource(XContentFactory.jsonBuilder() .startObject() .field("user", "kimchy") .field("postDate", new Date()) diff --git a/dspace-stats/src/main/java/org/dspace/statistics/ElasticSearchLoggerEventListener.java b/dspace-api/src/main/java/org/dspace/statistics/ElasticSearchLoggerEventListener.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/ElasticSearchLoggerEventListener.java rename to dspace-api/src/main/java/org/dspace/statistics/ElasticSearchLoggerEventListener.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/ObjectCount.java b/dspace-api/src/main/java/org/dspace/statistics/ObjectCount.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/ObjectCount.java rename to dspace-api/src/main/java/org/dspace/statistics/ObjectCount.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/SolrLogger.java b/dspace-api/src/main/java/org/dspace/statistics/SolrLogger.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/SolrLogger.java rename to dspace-api/src/main/java/org/dspace/statistics/SolrLogger.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/SolrLoggerUsageEventListener.java b/dspace-api/src/main/java/org/dspace/statistics/SolrLoggerUsageEventListener.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/SolrLoggerUsageEventListener.java rename to dspace-api/src/main/java/org/dspace/statistics/SolrLoggerUsageEventListener.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/StatisticsLoggingConsumer.java b/dspace-api/src/main/java/org/dspace/statistics/StatisticsLoggingConsumer.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/StatisticsLoggingConsumer.java rename to dspace-api/src/main/java/org/dspace/statistics/StatisticsLoggingConsumer.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/content/DSORepresentation.java b/dspace-api/src/main/java/org/dspace/statistics/content/DSORepresentation.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/content/DSORepresentation.java rename to dspace-api/src/main/java/org/dspace/statistics/content/DSORepresentation.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/content/DatasetDSpaceObjectGenerator.java b/dspace-api/src/main/java/org/dspace/statistics/content/DatasetDSpaceObjectGenerator.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/content/DatasetDSpaceObjectGenerator.java rename to dspace-api/src/main/java/org/dspace/statistics/content/DatasetDSpaceObjectGenerator.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/content/DatasetGenerator.java b/dspace-api/src/main/java/org/dspace/statistics/content/DatasetGenerator.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/content/DatasetGenerator.java rename to dspace-api/src/main/java/org/dspace/statistics/content/DatasetGenerator.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/content/DatasetSearchGenerator.java b/dspace-api/src/main/java/org/dspace/statistics/content/DatasetSearchGenerator.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/content/DatasetSearchGenerator.java rename to dspace-api/src/main/java/org/dspace/statistics/content/DatasetSearchGenerator.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/content/DatasetTimeGenerator.java b/dspace-api/src/main/java/org/dspace/statistics/content/DatasetTimeGenerator.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/content/DatasetTimeGenerator.java rename to dspace-api/src/main/java/org/dspace/statistics/content/DatasetTimeGenerator.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/content/DatasetTypeGenerator.java b/dspace-api/src/main/java/org/dspace/statistics/content/DatasetTypeGenerator.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/content/DatasetTypeGenerator.java rename to dspace-api/src/main/java/org/dspace/statistics/content/DatasetTypeGenerator.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/content/StatisticsBSAdapter.java b/dspace-api/src/main/java/org/dspace/statistics/content/StatisticsBSAdapter.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/content/StatisticsBSAdapter.java rename to dspace-api/src/main/java/org/dspace/statistics/content/StatisticsBSAdapter.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/content/StatisticsData.java b/dspace-api/src/main/java/org/dspace/statistics/content/StatisticsData.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/content/StatisticsData.java rename to dspace-api/src/main/java/org/dspace/statistics/content/StatisticsData.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/content/StatisticsDataSearches.java b/dspace-api/src/main/java/org/dspace/statistics/content/StatisticsDataSearches.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/content/StatisticsDataSearches.java rename to dspace-api/src/main/java/org/dspace/statistics/content/StatisticsDataSearches.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/content/StatisticsDataVisits.java b/dspace-api/src/main/java/org/dspace/statistics/content/StatisticsDataVisits.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/content/StatisticsDataVisits.java rename to dspace-api/src/main/java/org/dspace/statistics/content/StatisticsDataVisits.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/content/StatisticsDataWorkflow.java b/dspace-api/src/main/java/org/dspace/statistics/content/StatisticsDataWorkflow.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/content/StatisticsDataWorkflow.java rename to dspace-api/src/main/java/org/dspace/statistics/content/StatisticsDataWorkflow.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/content/StatisticsDisplay.java b/dspace-api/src/main/java/org/dspace/statistics/content/StatisticsDisplay.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/content/StatisticsDisplay.java rename to dspace-api/src/main/java/org/dspace/statistics/content/StatisticsDisplay.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/content/StatisticsListing.java b/dspace-api/src/main/java/org/dspace/statistics/content/StatisticsListing.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/content/StatisticsListing.java rename to dspace-api/src/main/java/org/dspace/statistics/content/StatisticsListing.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/content/StatisticsTable.java b/dspace-api/src/main/java/org/dspace/statistics/content/StatisticsTable.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/content/StatisticsTable.java rename to dspace-api/src/main/java/org/dspace/statistics/content/StatisticsTable.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/content/filter/StatisticsFilter.java b/dspace-api/src/main/java/org/dspace/statistics/content/filter/StatisticsFilter.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/content/filter/StatisticsFilter.java rename to dspace-api/src/main/java/org/dspace/statistics/content/filter/StatisticsFilter.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/content/filter/StatisticsSolrDateFilter.java b/dspace-api/src/main/java/org/dspace/statistics/content/filter/StatisticsSolrDateFilter.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/content/filter/StatisticsSolrDateFilter.java rename to dspace-api/src/main/java/org/dspace/statistics/content/filter/StatisticsSolrDateFilter.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/util/ApacheLogRobotsProcessor.java b/dspace-api/src/main/java/org/dspace/statistics/util/ApacheLogRobotsProcessor.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/util/ApacheLogRobotsProcessor.java rename to dspace-api/src/main/java/org/dspace/statistics/util/ApacheLogRobotsProcessor.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/util/ClassicDSpaceLogConverter.java b/dspace-api/src/main/java/org/dspace/statistics/util/ClassicDSpaceLogConverter.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/util/ClassicDSpaceLogConverter.java rename to dspace-api/src/main/java/org/dspace/statistics/util/ClassicDSpaceLogConverter.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/util/DnsLookup.java b/dspace-api/src/main/java/org/dspace/statistics/util/DnsLookup.java similarity index 96% rename from dspace-stats/src/main/java/org/dspace/statistics/util/DnsLookup.java rename to dspace-api/src/main/java/org/dspace/statistics/util/DnsLookup.java index 3bb1c34263..435bf03686 100644 --- a/dspace-stats/src/main/java/org/dspace/statistics/util/DnsLookup.java +++ b/dspace-api/src/main/java/org/dspace/statistics/util/DnsLookup.java @@ -1,47 +1,47 @@ -/** - * The contents of this file are subject to the license and copyright - * detailed in the LICENSE and NOTICE files at the root of the source - * tree and available online at - * - * http://www.dspace.org/license/ - */ -package org.dspace.statistics.util; - -import org.dspace.core.ConfigurationManager; -import org.xbill.DNS.*; - -import java.io.IOException; - -/** - * XBill DNS resolver to retrieve hostnames for client IP addresses. - * - * @author kevinvandevelde at atmire.com - * @author ben at atmire.com - */ -public class DnsLookup { - - public static String reverseDns(String hostIp) throws IOException { - Resolver res = new ExtendedResolver(); - - // set the timeout, defaults to 200 milliseconds - int timeout = ConfigurationManager.getIntProperty("usage-statistics", "resolver.timeout", 200); - res.setTimeout(0, timeout); - - Name name = ReverseMap.fromAddress(hostIp); - int type = Type.PTR; - int dclass = DClass.IN; - Record rec = Record.newRecord(name, type, dclass); - Message query = Message.newQuery(rec); - Message response = res.send(query); - - Record[] answers = response.getSectionArray(Section.ANSWER); - if (answers.length == 0) - { - return hostIp; - } - else - { - return answers[0].rdataToString(); - } - } -} +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.statistics.util; + +import org.dspace.core.ConfigurationManager; +import org.xbill.DNS.*; + +import java.io.IOException; + +/** + * XBill DNS resolver to retrieve hostnames for client IP addresses. + * + * @author kevinvandevelde at atmire.com + * @author ben at atmire.com + */ +public class DnsLookup { + + public static String reverseDns(String hostIp) throws IOException { + Resolver res = new ExtendedResolver(); + + // set the timeout, defaults to 200 milliseconds + int timeout = ConfigurationManager.getIntProperty("usage-statistics", "resolver.timeout", 200); + res.setTimeout(0, timeout); + + Name name = ReverseMap.fromAddress(hostIp); + int type = Type.PTR; + int dclass = DClass.IN; + Record rec = Record.newRecord(name, type, dclass); + Message query = Message.newQuery(rec); + Message response = res.send(query); + + Record[] answers = response.getSectionArray(Section.ANSWER); + if (answers.length == 0) + { + return hostIp; + } + else + { + return answers[0].rdataToString(); + } + } +} diff --git a/dspace-stats/src/main/java/org/dspace/statistics/util/IPTable.java b/dspace-api/src/main/java/org/dspace/statistics/util/IPTable.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/util/IPTable.java rename to dspace-api/src/main/java/org/dspace/statistics/util/IPTable.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/util/LocationUtils.java b/dspace-api/src/main/java/org/dspace/statistics/util/LocationUtils.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/util/LocationUtils.java rename to dspace-api/src/main/java/org/dspace/statistics/util/LocationUtils.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/util/SpiderDetector.java b/dspace-api/src/main/java/org/dspace/statistics/util/SpiderDetector.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/util/SpiderDetector.java rename to dspace-api/src/main/java/org/dspace/statistics/util/SpiderDetector.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/util/StatisticsClient.java b/dspace-api/src/main/java/org/dspace/statistics/util/StatisticsClient.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/util/StatisticsClient.java rename to dspace-api/src/main/java/org/dspace/statistics/util/StatisticsClient.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/util/StatisticsDataGenerator.java b/dspace-api/src/main/java/org/dspace/statistics/util/StatisticsDataGenerator.java similarity index 96% rename from dspace-stats/src/main/java/org/dspace/statistics/util/StatisticsDataGenerator.java rename to dspace-api/src/main/java/org/dspace/statistics/util/StatisticsDataGenerator.java index 6c99cfb159..ddd4392bce 100644 --- a/dspace-stats/src/main/java/org/dspace/statistics/util/StatisticsDataGenerator.java +++ b/dspace-api/src/main/java/org/dspace/statistics/util/StatisticsDataGenerator.java @@ -1,427 +1,427 @@ -/** - * The contents of this file are subject to the license and copyright - * detailed in the LICENSE and NOTICE files at the root of the source - * tree and available online at - * - * http://www.dspace.org/license/ - */ -package org.dspace.statistics.util; - -import org.apache.commons.cli.*; -import org.apache.commons.lang.time.DateFormatUtils; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer; -import org.dspace.core.Context; -import org.dspace.core.Constants; -import org.dspace.core.ConfigurationManager; -import org.dspace.content.DSpaceObject; -import org.dspace.content.Bitstream; -import org.dspace.content.DCValue; -import org.dspace.content.Item; -import org.dspace.eperson.EPerson; -import org.dspace.statistics.SolrLogger; - -import java.util.Date; -import java.util.Map; -import java.text.SimpleDateFormat; - -import com.maxmind.geoip.LookupService; -import com.maxmind.geoip.Location; - -/** - * Test class to generate random statistics data. - * Used for load testing of searches. Inputs are slow - * due to inefficient randomizer. - * - * @author kevinvandevelde at atmire.com - * @author ben at atmire.com - */ -public class StatisticsDataGenerator { - public static void main(String[] args) throws Exception { - CommandLineParser parser = new PosixParser(); - - Options options = new Options(); - options.addOption("n", "nrlogs", true, - "type: nr of logs to be generated"); - options.addOption("s", "startDate", true, - "type: the start date from which we start generating our logs"); - options.addOption("e", "endDate", true, - "type: the end date from which we start generating our logs"); - options.addOption("a", "cms", true, "The starting id of our community"); - options.addOption("b", "cme", true, "The end id of our community"); - options - .addOption("c", "cls", true, - "The starting id of our collection"); - options.addOption("d", "cle", true, "The end if of our collection"); - options.addOption("f", "is", true, "The starting id of our item"); - options.addOption("g", "ie", true, "The end id of our item"); - options.addOption("h", "bs", true, "The starting id of our bitstream"); - options.addOption("i", "be", true, "The end id of our bitstream"); - options.addOption("j", "ps", true, "The starting id of our epersons"); - options.addOption("k", "pe", true, "The end id of our epersons"); - - CommandLine line = parser.parse(options, args); - - int nrLogs; - long startDate; - long endDate; - long commStartId; - long commEndId; - long collStartId; - long collEndId; - long itemStartId; - long itemEndId; - long bitStartId; - long bitEndId; - long epersonStartId; - long epersonEndId; - - if (line.hasOption("n")) - { - nrLogs = Integer.parseInt(line.getOptionValue("n")); - } - else { - System.out - .println("We need to know how many logs we need to create"); - return; - } - if (line.hasOption("s")) { - startDate = getDateInMiliseconds(line.getOptionValue("s")); - } else - { - startDate = getDateInMiliseconds("01/01/2006"); - } - if (line.hasOption("e")) { - endDate = getDateInMiliseconds(line.getOptionValue("e")); - } else - { - endDate = new Date().getTime(); - } - - if (line.hasOption("a")) - { - commStartId = Long.parseLong(line.getOptionValue("a")); - } - else - { - return; - } - - if (line.hasOption("b")) - { - commEndId = Long.parseLong(line.getOptionValue("b")); - } - else - { - return; - } - if (line.hasOption("c")) - { - collStartId = Long.parseLong(line.getOptionValue("c")); - } - else - { - return; - } - if (line.hasOption("d")) - { - collEndId = Long.parseLong(line.getOptionValue("d")); - } - else - { - return; - } - if (line.hasOption("f")) - { - itemStartId = Long.parseLong(line.getOptionValue("f")); - } - else - { - return; - } - if (line.hasOption("g")) - { - itemEndId = Long.parseLong(line.getOptionValue("g")); - } - else - { - return; - } - if (line.hasOption("h")) - { - bitStartId = Long.parseLong(line.getOptionValue("h")); - } - else - { - return; - } - if (line.hasOption("i")) - { - bitEndId = Long.parseLong(line.getOptionValue("i")); - } - else - { - return; - } - if (line.hasOption("j")) - { - epersonStartId = Long.parseLong(line.getOptionValue("j")); - } - else - { - return; - } - if (line.hasOption("k")) - { - epersonEndId = Long.parseLong(line.getOptionValue("k")); - } - else - { - return; - } - - // Get the max id range - long maxIdTotal = Math.max(commEndId, collEndId); - maxIdTotal = Math.max(maxIdTotal, itemEndId); - maxIdTotal = Math.max(maxIdTotal, bitEndId); - - // We got 3/4 chance the person visting the dso is not logged in - epersonEndId *= 4; - - // We got all our parameters now get the rest - Context context = new Context(); - // Find our solr server - CommonsHttpSolrServer solr = new CommonsHttpSolrServer( - ConfigurationManager.getProperty("solr-statistics", "server")); - solr.deleteByQuery("*:*"); - solr.commit(); - - Map metadataStorageInfo = SolrLogger.getMetadataStorageInfo(); - - String prevIp = null; - String dbfile = ConfigurationManager.getProperty("usage-statistics", "dbfile"); - LookupService cl = new LookupService(dbfile, - LookupService.GEOIP_STANDARD); - int countryErrors = 0; - for (int i = 0; i < nrLogs; i++) { - String ip = ""; - Date time; - String continent; - String countryCode; - float longitude; - float latitude; - String city; - - // 1. Generate an ip for our user - StringBuilder ipBuilder = new StringBuilder(); - for (int j = 0; j < 4; j++) { - ipBuilder.append(getRandomNumberInRange(0, 254)); - if (j != 3) - { - ipBuilder.append("."); - } - } - ip = ipBuilder.toString(); - - // 2 Depending on our ip get all the location info - Location location; - try { - location = cl.getLocation(ip); - } catch (Exception e) { - location = null; - } - if (location == null) { - // If we haven't got a prev ip this is pretty useless so move on - // to the next one - if (prevIp == null) - { - continue; - } - ip = prevIp; - location = cl.getLocation(ip); - } - - city = location.city; - countryCode = location.countryCode; - longitude = location.longitude; - latitude = location.latitude; - try { - continent = LocationUtils.getContinentCode(countryCode); - } catch (Exception e) { - // We could get an error if our country == Europa this doesn't - // matter for generating statistics so ignore it - System.out.println("COUNTRY ERROR: " + countryCode); - countryErrors++; - continue; - } - - // 3. Generate a date that the object was visited - time = new Date(getRandomNumberInRange(startDate, endDate)); - - // 4. Get our dspaceobject we are supposed to be working on - // We got mostly item views so lets say we got 1/2 chance that we - // got an item view - // What type have we got (PS: I know we haven't got 5 as a dso type - // we can log but it is used so our item gets move traffic) - int type = (int) getRandomNumberInRange(0, 8); - if (type == Constants.BUNDLE || type >= 5) - { - type = Constants.ITEM; - } - - int dsoId = -1; - // Now we need to find a valid id - switch (type) { - case Constants.COMMUNITY: - dsoId = (int) getRandomNumberInRange(commStartId, commEndId); - break; - case Constants.COLLECTION: - dsoId = (int) getRandomNumberInRange(collStartId, collEndId); - break; - case Constants.ITEM: - dsoId = (int) getRandomNumberInRange(itemStartId, itemEndId); - break; - case Constants.BITSTREAM: - dsoId = (int) getRandomNumberInRange(bitStartId, bitEndId); - break; - } - // Now find our dso - DSpaceObject dso = DSpaceObject.find(context, type, dsoId); - if (dso instanceof Bitstream) { - Bitstream bit = (Bitstream) dso; - if (bit.getFormat().isInternal()) { - dso = null; - } - } - // Make sure we got a dso - boolean substract = false; - while (dso == null) { - // If our dsoId gets higher then our maxIdtotal we need to lower - // to find a valid id - if (dsoId == maxIdTotal) - { - substract = true; - } - - if (substract) - { - dsoId--; - } - else - { - dsoId++; - } - - dso = DSpaceObject.find(context, type, dsoId); - if (dso instanceof Bitstream) { - Bitstream bit = (Bitstream) dso; - if (bit.getFormat().isInternal()) { - dso = null; - } - } - // System.out.println("REFIND"); - } - // Find the person who is visting us - int epersonId = (int) getRandomNumberInRange(epersonStartId, epersonEndId); - EPerson eperson = EPerson.find(context, epersonId); - if (eperson == null) - { - epersonId = -1; - } - - // System.out.println(ip); - // System.out.println(country + " " + - // LocationUtils.getCountryName(countryCode)); - - // Resolve the dns - String dns = null; - try { - dns = DnsLookup.reverseDns(ip); - } catch (Exception e) { - - } - - System.out.println(ip); - System.out.println(dns); - - // Save it in our server - SolrInputDocument doc1 = new SolrInputDocument(); - doc1.addField("ip", ip); - doc1.addField("type", dso.getType()); - doc1.addField("id", dso.getID()); - doc1.addField("time", DateFormatUtils.format(time, - SolrLogger.DATE_FORMAT_8601)); - doc1.addField("continent", continent); - // doc1.addField("country", country); - doc1.addField("countryCode", countryCode); - doc1.addField("city", city); - doc1.addField("latitude", latitude); - doc1.addField("longitude", longitude); - if (epersonId > 0) - { - doc1.addField("epersonid", epersonId); - } - if (dns != null) - { - doc1.addField("dns", dns.toLowerCase()); - } - - if (dso instanceof Item) { - Item item = (Item) dso; - // Store the metadata - for (Map.Entry entry : metadataStorageInfo.entrySet()) - { - String dcField = entry.getValue(); - - DCValue[] vals = item.getMetadata(dcField.split("\\.")[0], - dcField.split("\\.")[1], dcField.split("\\.")[2], - Item.ANY); - for (DCValue val1 : vals) { - String val = val1.value; - doc1.addField(entry.getKey(), val); - doc1.addField(entry.getKey() + "_search", val.toLowerCase()); - } - } - } - - SolrLogger.storeParents(doc1, dso); - - solr.add(doc1); - - // Make sure we have a previous ip - prevIp = ip; - } - System.out.println("Nr of countryErrors: " + countryErrors); - // Commit at the end cause it takes a while - solr.commit(); - } - - /** - * Method returns a random integer between the given int - * - * @param min - * the random number must be greater or equal to this - * @param max - * the random number must be smaller or equal to this - * @return a random in - */ - private static long getRandomNumberInRange(long min, long max) { - return min + (long) (Math.random() * ((max - min) + 1)); - } - - /** - * Method to get the miliseconds from a datestring - * - * @param dateString - * the string containing our date in a string - * @return the nr of miliseconds in the given datestring - * @throws java.text.ParseException - * should not happen - */ - private static long getDateInMiliseconds(String dateString) - throws java.text.ParseException { - SimpleDateFormat formatter = new SimpleDateFormat("dd/MM/yyyy"); - return formatter.parse(dateString).getTime(); - } - -} +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.statistics.util; + +import org.apache.commons.cli.*; +import org.apache.commons.lang.time.DateFormatUtils; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer; +import org.dspace.core.Context; +import org.dspace.core.Constants; +import org.dspace.core.ConfigurationManager; +import org.dspace.content.DSpaceObject; +import org.dspace.content.Bitstream; +import org.dspace.content.DCValue; +import org.dspace.content.Item; +import org.dspace.eperson.EPerson; +import org.dspace.statistics.SolrLogger; + +import java.util.Date; +import java.util.Map; +import java.text.SimpleDateFormat; + +import com.maxmind.geoip.LookupService; +import com.maxmind.geoip.Location; + +/** + * Test class to generate random statistics data. + * Used for load testing of searches. Inputs are slow + * due to inefficient randomizer. + * + * @author kevinvandevelde at atmire.com + * @author ben at atmire.com + */ +public class StatisticsDataGenerator { + public static void main(String[] args) throws Exception { + CommandLineParser parser = new PosixParser(); + + Options options = new Options(); + options.addOption("n", "nrlogs", true, + "type: nr of logs to be generated"); + options.addOption("s", "startDate", true, + "type: the start date from which we start generating our logs"); + options.addOption("e", "endDate", true, + "type: the end date from which we start generating our logs"); + options.addOption("a", "cms", true, "The starting id of our community"); + options.addOption("b", "cme", true, "The end id of our community"); + options + .addOption("c", "cls", true, + "The starting id of our collection"); + options.addOption("d", "cle", true, "The end if of our collection"); + options.addOption("f", "is", true, "The starting id of our item"); + options.addOption("g", "ie", true, "The end id of our item"); + options.addOption("h", "bs", true, "The starting id of our bitstream"); + options.addOption("i", "be", true, "The end id of our bitstream"); + options.addOption("j", "ps", true, "The starting id of our epersons"); + options.addOption("k", "pe", true, "The end id of our epersons"); + + CommandLine line = parser.parse(options, args); + + int nrLogs; + long startDate; + long endDate; + long commStartId; + long commEndId; + long collStartId; + long collEndId; + long itemStartId; + long itemEndId; + long bitStartId; + long bitEndId; + long epersonStartId; + long epersonEndId; + + if (line.hasOption("n")) + { + nrLogs = Integer.parseInt(line.getOptionValue("n")); + } + else { + System.out + .println("We need to know how many logs we need to create"); + return; + } + if (line.hasOption("s")) { + startDate = getDateInMiliseconds(line.getOptionValue("s")); + } else + { + startDate = getDateInMiliseconds("01/01/2006"); + } + if (line.hasOption("e")) { + endDate = getDateInMiliseconds(line.getOptionValue("e")); + } else + { + endDate = new Date().getTime(); + } + + if (line.hasOption("a")) + { + commStartId = Long.parseLong(line.getOptionValue("a")); + } + else + { + return; + } + + if (line.hasOption("b")) + { + commEndId = Long.parseLong(line.getOptionValue("b")); + } + else + { + return; + } + if (line.hasOption("c")) + { + collStartId = Long.parseLong(line.getOptionValue("c")); + } + else + { + return; + } + if (line.hasOption("d")) + { + collEndId = Long.parseLong(line.getOptionValue("d")); + } + else + { + return; + } + if (line.hasOption("f")) + { + itemStartId = Long.parseLong(line.getOptionValue("f")); + } + else + { + return; + } + if (line.hasOption("g")) + { + itemEndId = Long.parseLong(line.getOptionValue("g")); + } + else + { + return; + } + if (line.hasOption("h")) + { + bitStartId = Long.parseLong(line.getOptionValue("h")); + } + else + { + return; + } + if (line.hasOption("i")) + { + bitEndId = Long.parseLong(line.getOptionValue("i")); + } + else + { + return; + } + if (line.hasOption("j")) + { + epersonStartId = Long.parseLong(line.getOptionValue("j")); + } + else + { + return; + } + if (line.hasOption("k")) + { + epersonEndId = Long.parseLong(line.getOptionValue("k")); + } + else + { + return; + } + + // Get the max id range + long maxIdTotal = Math.max(commEndId, collEndId); + maxIdTotal = Math.max(maxIdTotal, itemEndId); + maxIdTotal = Math.max(maxIdTotal, bitEndId); + + // We got 3/4 chance the person visting the dso is not logged in + epersonEndId *= 4; + + // We got all our parameters now get the rest + Context context = new Context(); + // Find our solr server + CommonsHttpSolrServer solr = new CommonsHttpSolrServer( + ConfigurationManager.getProperty("solr-statistics", "server")); + solr.deleteByQuery("*:*"); + solr.commit(); + + Map metadataStorageInfo = SolrLogger.getMetadataStorageInfo(); + + String prevIp = null; + String dbfile = ConfigurationManager.getProperty("usage-statistics", "dbfile"); + LookupService cl = new LookupService(dbfile, + LookupService.GEOIP_STANDARD); + int countryErrors = 0; + for (int i = 0; i < nrLogs; i++) { + String ip = ""; + Date time; + String continent; + String countryCode; + float longitude; + float latitude; + String city; + + // 1. Generate an ip for our user + StringBuilder ipBuilder = new StringBuilder(); + for (int j = 0; j < 4; j++) { + ipBuilder.append(getRandomNumberInRange(0, 254)); + if (j != 3) + { + ipBuilder.append("."); + } + } + ip = ipBuilder.toString(); + + // 2 Depending on our ip get all the location info + Location location; + try { + location = cl.getLocation(ip); + } catch (Exception e) { + location = null; + } + if (location == null) { + // If we haven't got a prev ip this is pretty useless so move on + // to the next one + if (prevIp == null) + { + continue; + } + ip = prevIp; + location = cl.getLocation(ip); + } + + city = location.city; + countryCode = location.countryCode; + longitude = location.longitude; + latitude = location.latitude; + try { + continent = LocationUtils.getContinentCode(countryCode); + } catch (Exception e) { + // We could get an error if our country == Europa this doesn't + // matter for generating statistics so ignore it + System.out.println("COUNTRY ERROR: " + countryCode); + countryErrors++; + continue; + } + + // 3. Generate a date that the object was visited + time = new Date(getRandomNumberInRange(startDate, endDate)); + + // 4. Get our dspaceobject we are supposed to be working on + // We got mostly item views so lets say we got 1/2 chance that we + // got an item view + // What type have we got (PS: I know we haven't got 5 as a dso type + // we can log but it is used so our item gets move traffic) + int type = (int) getRandomNumberInRange(0, 8); + if (type == Constants.BUNDLE || type >= 5) + { + type = Constants.ITEM; + } + + int dsoId = -1; + // Now we need to find a valid id + switch (type) { + case Constants.COMMUNITY: + dsoId = (int) getRandomNumberInRange(commStartId, commEndId); + break; + case Constants.COLLECTION: + dsoId = (int) getRandomNumberInRange(collStartId, collEndId); + break; + case Constants.ITEM: + dsoId = (int) getRandomNumberInRange(itemStartId, itemEndId); + break; + case Constants.BITSTREAM: + dsoId = (int) getRandomNumberInRange(bitStartId, bitEndId); + break; + } + // Now find our dso + DSpaceObject dso = DSpaceObject.find(context, type, dsoId); + if (dso instanceof Bitstream) { + Bitstream bit = (Bitstream) dso; + if (bit.getFormat().isInternal()) { + dso = null; + } + } + // Make sure we got a dso + boolean substract = false; + while (dso == null) { + // If our dsoId gets higher then our maxIdtotal we need to lower + // to find a valid id + if (dsoId == maxIdTotal) + { + substract = true; + } + + if (substract) + { + dsoId--; + } + else + { + dsoId++; + } + + dso = DSpaceObject.find(context, type, dsoId); + if (dso instanceof Bitstream) { + Bitstream bit = (Bitstream) dso; + if (bit.getFormat().isInternal()) { + dso = null; + } + } + // System.out.println("REFIND"); + } + // Find the person who is visting us + int epersonId = (int) getRandomNumberInRange(epersonStartId, epersonEndId); + EPerson eperson = EPerson.find(context, epersonId); + if (eperson == null) + { + epersonId = -1; + } + + // System.out.println(ip); + // System.out.println(country + " " + + // LocationUtils.getCountryName(countryCode)); + + // Resolve the dns + String dns = null; + try { + dns = DnsLookup.reverseDns(ip); + } catch (Exception e) { + + } + + System.out.println(ip); + System.out.println(dns); + + // Save it in our server + SolrInputDocument doc1 = new SolrInputDocument(); + doc1.addField("ip", ip); + doc1.addField("type", dso.getType()); + doc1.addField("id", dso.getID()); + doc1.addField("time", DateFormatUtils.format(time, + SolrLogger.DATE_FORMAT_8601)); + doc1.addField("continent", continent); + // doc1.addField("country", country); + doc1.addField("countryCode", countryCode); + doc1.addField("city", city); + doc1.addField("latitude", latitude); + doc1.addField("longitude", longitude); + if (epersonId > 0) + { + doc1.addField("epersonid", epersonId); + } + if (dns != null) + { + doc1.addField("dns", dns.toLowerCase()); + } + + if (dso instanceof Item) { + Item item = (Item) dso; + // Store the metadata + for (Map.Entry entry : metadataStorageInfo.entrySet()) + { + String dcField = entry.getValue(); + + DCValue[] vals = item.getMetadata(dcField.split("\\.")[0], + dcField.split("\\.")[1], dcField.split("\\.")[2], + Item.ANY); + for (DCValue val1 : vals) { + String val = val1.value; + doc1.addField(entry.getKey(), val); + doc1.addField(entry.getKey() + "_search", val.toLowerCase()); + } + } + } + + SolrLogger.storeParents(doc1, dso); + + solr.add(doc1); + + // Make sure we have a previous ip + prevIp = ip; + } + System.out.println("Nr of countryErrors: " + countryErrors); + // Commit at the end cause it takes a while + solr.commit(); + } + + /** + * Method returns a random integer between the given int + * + * @param min + * the random number must be greater or equal to this + * @param max + * the random number must be smaller or equal to this + * @return a random in + */ + private static long getRandomNumberInRange(long min, long max) { + return min + (long) (Math.random() * ((max - min) + 1)); + } + + /** + * Method to get the miliseconds from a datestring + * + * @param dateString + * the string containing our date in a string + * @return the nr of miliseconds in the given datestring + * @throws java.text.ParseException + * should not happen + */ + private static long getDateInMiliseconds(String dateString) + throws java.text.ParseException { + SimpleDateFormat formatter = new SimpleDateFormat("dd/MM/yyyy"); + return formatter.parse(dateString).getTime(); + } + +} diff --git a/dspace-stats/src/main/java/org/dspace/statistics/util/StatisticsImporter.java b/dspace-api/src/main/java/org/dspace/statistics/util/StatisticsImporter.java similarity index 100% rename from dspace-stats/src/main/java/org/dspace/statistics/util/StatisticsImporter.java rename to dspace-api/src/main/java/org/dspace/statistics/util/StatisticsImporter.java diff --git a/dspace-stats/src/main/java/org/dspace/statistics/util/StatisticsImporterElasticSearch.java b/dspace-api/src/main/java/org/dspace/statistics/util/StatisticsImporterElasticSearch.java similarity index 99% rename from dspace-stats/src/main/java/org/dspace/statistics/util/StatisticsImporterElasticSearch.java rename to dspace-api/src/main/java/org/dspace/statistics/util/StatisticsImporterElasticSearch.java index 7815475f70..931fbc339f 100644 --- a/dspace-stats/src/main/java/org/dspace/statistics/util/StatisticsImporterElasticSearch.java +++ b/dspace-api/src/main/java/org/dspace/statistics/util/StatisticsImporterElasticSearch.java @@ -239,7 +239,7 @@ public class StatisticsImporterElasticSearch { //TODO Is there any way to reuse ElasticSearchLogger.post() ? // Save it in our server - XContentBuilder postBuilder = jsonBuilder().startObject() + XContentBuilder postBuilder = XContentFactory.jsonBuilder().startObject() .field("id", dso.getID()) .field("typeIndex", dso.getType()) .field("type", dso.getTypeText()) diff --git a/dspace-stats/src/main/resources/org/dspace/statistics/util/continent-names.properties b/dspace-api/src/main/resources/org/dspace/statistics/util/continent-names.properties similarity index 100% rename from dspace-stats/src/main/resources/org/dspace/statistics/util/continent-names.properties rename to dspace-api/src/main/resources/org/dspace/statistics/util/continent-names.properties diff --git a/dspace-stats/src/main/resources/org/dspace/statistics/util/country-continent-codes.properties b/dspace-api/src/main/resources/org/dspace/statistics/util/country-continent-codes.properties similarity index 100% rename from dspace-stats/src/main/resources/org/dspace/statistics/util/country-continent-codes.properties rename to dspace-api/src/main/resources/org/dspace/statistics/util/country-continent-codes.properties diff --git a/dspace-stats/src/test/java/org/dspace/statistics/util/TestLocationUtils.java b/dspace-api/src/test/java/org/dspace/statistics/util/TestLocationUtils.java similarity index 100% rename from dspace-stats/src/test/java/org/dspace/statistics/util/TestLocationUtils.java rename to dspace-api/src/test/java/org/dspace/statistics/util/TestLocationUtils.java diff --git a/dspace-jspui/dspace-jspui-api/pom.xml b/dspace-jspui/dspace-jspui-api/pom.xml index 872e47ebb1..0bfc464076 100644 --- a/dspace-jspui/dspace-jspui-api/pom.xml +++ b/dspace-jspui/dspace-jspui-api/pom.xml @@ -38,10 +38,6 @@ org.dspace dspace-services-utils - - org.dspace - dspace-stats - commons-validator commons-validator diff --git a/dspace-stats/src/main/java/org/dspace/statistics/content/package-info.java b/dspace-stats/src/main/java/org/dspace/statistics/content/package-info.java deleted file mode 100644 index ee663da9f1..0000000000 --- a/dspace-stats/src/main/java/org/dspace/statistics/content/package-info.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * The contents of this file are subject to the license and copyright - * detailed in the LICENSE and NOTICE files at the root of the source - * tree and available online at - * - * http://www.dspace.org/license/ - */ -/** - * Build queries to the statistical subsystem and create UI elements from the - * results. The underlying SOLR engine uses a text-based query language. The - * classes here map a structure of constraint objects into that language. - *

- * {@link org.dspace.statistics.content.StatisticsDataVisits} is somewhat like a - * factory for statistical queries. An instance is customized with - * DatasetGenerator instances to specify interesting facets of the data and with - * filters to specify TBS. The "factory methods" then produce arrays of values - * meeting the factory's criteria, either raw or formatted for presentation. - *

- * DatasetGenerator subclasses are available for constraining the results to a - * given DSpaceObject, object type, and time interval. - *

- * A StatisticsDataVisits object can be wrapped in a - * {@link org.dspace.statistics.content.StatisticsDisplay} - * subclass to format its content as a list or a table. - */ - -package org.dspace.statistics.content; diff --git a/dspace-xmlui/dspace-xmlui-api/pom.xml b/dspace-xmlui/dspace-xmlui-api/pom.xml index 419a9c055b..fcf198db11 100644 --- a/dspace-xmlui/dspace-xmlui-api/pom.xml +++ b/dspace-xmlui/dspace-xmlui-api/pom.xml @@ -43,10 +43,6 @@ org.dspace dspace-services-utils - - org.dspace - dspace-stats - javax.servlet servlet-api diff --git a/dspace-xmlui/dspace-xmlui-api/src/main/java/org/dspace/app/xmlui/aspect/statistics/StatisticsSearchTransformer.java b/dspace-xmlui/dspace-xmlui-api/src/main/java/org/dspace/app/xmlui/aspect/statistics/StatisticsSearchTransformer.java index 458737b7b0..200d69605d 100644 --- a/dspace-xmlui/dspace-xmlui-api/src/main/java/org/dspace/app/xmlui/aspect/statistics/StatisticsSearchTransformer.java +++ b/dspace-xmlui/dspace-xmlui-api/src/main/java/org/dspace/app/xmlui/aspect/statistics/StatisticsSearchTransformer.java @@ -11,7 +11,6 @@ import org.apache.cocoon.ProcessingException; import org.apache.cocoon.environment.ObjectModelHelper; import org.apache.cocoon.environment.Request; import org.dspace.app.xmlui.utils.HandleUtil; -import org.dspace.app.xmlui.utils.UIException; import org.dspace.app.xmlui.wing.Message; import org.dspace.app.xmlui.wing.WingException; import org.dspace.app.xmlui.wing.element.Body; diff --git a/dspace-xmlui/dspace-xmlui-api/src/main/java/org/dspace/app/xmlui/aspect/statistics/StatisticsTransformer.java b/dspace-xmlui/dspace-xmlui-api/src/main/java/org/dspace/app/xmlui/aspect/statistics/StatisticsTransformer.java index e05457ff6b..91e4821716 100644 --- a/dspace-xmlui/dspace-xmlui-api/src/main/java/org/dspace/app/xmlui/aspect/statistics/StatisticsTransformer.java +++ b/dspace-xmlui/dspace-xmlui-api/src/main/java/org/dspace/app/xmlui/aspect/statistics/StatisticsTransformer.java @@ -7,7 +7,6 @@ */ package org.dspace.app.xmlui.aspect.statistics; -import com.google.gson.Gson; import org.apache.log4j.Logger; import org.apache.solr.client.solrj.SolrServerException; import org.dspace.app.xmlui.cocoon.AbstractDSpaceTransformer; @@ -21,20 +20,13 @@ import org.dspace.content.DSpaceObject; import org.dspace.core.Constants; import org.dspace.core.Context; import org.dspace.statistics.Dataset; -import org.dspace.statistics.ObjectCount; -import org.dspace.statistics.content.*; -import org.dspace.storage.rdbms.DatabaseManager; -import org.dspace.storage.rdbms.TableRow; -import org.dspace.storage.rdbms.TableRowIterator; import org.xml.sax.SAXException; import java.io.IOException; import java.sql.SQLException; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.Calendar; import java.util.Date; -import java.util.Locale; public class StatisticsTransformer extends AbstractDSpaceTransformer { diff --git a/dspace-xmlui/dspace-xmlui-api/src/main/java/org/dspace/app/xmlui/aspect/statisticsElasticSearch/ElasticSearchStatsViewer.java b/dspace-xmlui/dspace-xmlui-api/src/main/java/org/dspace/app/xmlui/aspect/statisticsElasticSearch/ElasticSearchStatsViewer.java index f6ee9346ff..777cbc2e12 100644 --- a/dspace-xmlui/dspace-xmlui-api/src/main/java/org/dspace/app/xmlui/aspect/statisticsElasticSearch/ElasticSearchStatsViewer.java +++ b/dspace-xmlui/dspace-xmlui-api/src/main/java/org/dspace/app/xmlui/aspect/statisticsElasticSearch/ElasticSearchStatsViewer.java @@ -10,7 +10,6 @@ package org.dspace.app.xmlui.aspect.statisticsElasticSearch; import org.apache.cocoon.environment.ObjectModelHelper; import org.apache.cocoon.environment.Request; import org.apache.log4j.Logger; -import org.dspace.app.xmlui.aspect.statistics.StatisticsTransformer; import org.dspace.app.xmlui.cocoon.AbstractDSpaceTransformer; import org.dspace.app.xmlui.utils.HandleUtil; import org.dspace.app.xmlui.wing.Message; diff --git a/dspace/modules/additions/pom.xml b/dspace/modules/additions/pom.xml index e854a4d535..93be2466bc 100644 --- a/dspace/modules/additions/pom.xml +++ b/dspace/modules/additions/pom.xml @@ -79,10 +79,6 @@ org.dspace dspace-services-utils - - org.dspace - dspace-stats - org.dspace dspace-discovery-provider diff --git a/pom.xml b/pom.xml index 65d4fb1bd2..005cca45ce 100644 --- a/pom.xml +++ b/pom.xml @@ -223,19 +223,6 @@ - - - dspace-stats - - true - - - dspace-stats - - - @@ -547,11 +534,6 @@ dspace-services-utils 3.0-SNAPSHOT - - org.dspace - dspace-stats - 3.0-SNAPSHOT - org.dspace