hazza/DSpace
1
0
Fork 0
mirror of https://github.com/DSpace/DSpace.git synced 2025-10-07 01:54:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #63 from peterdietz/elastic-search-stats

Browse Source 
DS-1241 Statistics implementation in Elastic Search
This commit is contained in:
Hardy Pottinger
2012-09-19 13:42:33 -07:00
parent 59cedf1018 27d10c0734
commit 53f1fa3913
37 changed files with 3351 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
dspace-api/src/main/java/org/dspace/content/DSpaceObject.java
View File

@@ -7,13 +7,13 @@
*/
package org.dspace.content;
import java.sql.SQLException;
import org.dspace.core.Constants;
import org.dspace.core.Context;
import org.dspace.eperson.EPerson;
import org.dspace.eperson.Group;
import java.sql.SQLException;
/**
* Abstract base class for DSpace objects
*/
@@ -65,6 +65,15 @@ public abstract class DSpaceObject
*/
public abstract int getType();
/**
* Provide the text name of the type of this DSpaceObject. It is most likely all uppercase.
* @return Object type as text
*/
public String getTypeText()
{
return Constants.typeText[this.getType()];
}
/**
* Get the internal ID (database primary key) of this object
*

2
dspace-jspui/dspace-jspui-api/src/main/java/org/dspace/app/webui/servlet/DisplayStatisticsServlet.java
View File

@@ -55,7 +55,7 @@ public class DisplayStatisticsServlet extends DSpaceServlet
{
// is the statistics data publically viewable?
boolean privatereport = ConfigurationManager.getBooleanProperty("solr-statistics", "authorization.admin");
boolean privatereport = ConfigurationManager.getBooleanProperty("usage-statistics", "authorization.admin");
// is the user a member of the Administrator (1) group?
boolean admin = Group.isMember(context, 1);

24
dspace-stats/pom.xml
View File

@@ -141,7 +141,17 @@
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-solrj</artifactId>
<version>3.3.0</version>
<version>${lucene.version}</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.dspace.dependencies</groupId>
@@ -194,11 +204,23 @@
<version>4.8.2</version>
<type>jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>0.18.6</version>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
</dependency>
<!-- Gson: Java to Json conversion -->
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.1</version>
<scope>compile</scope>
</dependency>
</dependencies>
<reporting>

71
dspace-stats/src/main/java/org/dspace/statistics/DataTermsFacet.java Normal file
View File

@@ -0,0 +1,71 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.statistics;
import com.google.gson.Gson;
import java.util.ArrayList;
import java.util.List;
/**
* A neutral data object to hold data for statistics.
*
*/
public class DataTermsFacet {
private List<TermsFacet> terms;
public DataTermsFacet() {
terms = new ArrayList<TermsFacet>();
}
public void addTermFacet(TermsFacet termsFacet ) {
terms.add(termsFacet);
}
/**
* Render this data object into JSON format.
*
* An example of the output could be of the format:
* [{"term":"247166","count":10},{"term":"247168","count":6}]
* @return
*/
public String toJson() {
Gson gson = new Gson();
return gson.toJson(terms);
}
public static class TermsFacet {
private String term;
private Integer count;
public TermsFacet(String term, Integer count) {
setTerm(term);
setCount(count);
}
public String getTerm() {
return term;
}
public void setTerm(String term) {
this.term = term;
}
public Integer getCount() {
return count;
}
public void setCount(Integer count) {
this.count = count;
}
}
}

562
dspace-stats/src/main/java/org/dspace/statistics/ElasticSearchLogger.java Normal file
View File

@@ -0,0 +1,562 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.statistics;
import com.maxmind.geoip.Location;
import com.maxmind.geoip.LookupService;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.lang.time.DateFormatUtils;
import org.apache.log4j.Logger;
import org.dspace.content.*;
import org.dspace.content.Collection;
import org.dspace.core.ConfigurationManager;
import org.dspace.core.Constants;
import org.dspace.eperson.EPerson;
import org.dspace.statistics.util.DnsLookup;
import org.dspace.statistics.util.LocationUtils;
import org.dspace.statistics.util.SpiderDetector;
import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.admin.indices.exists.IndicesExistsRequest;
import org.elasticsearch.action.admin.indices.exists.IndicesExistsResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.action.admin.indices.mapping.put.PutMappingRequestBuilder;
import org.elasticsearch.client.action.index.IndexRequestBuilder;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeBuilder;
import javax.servlet.http.HttpServletRequest;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.sql.SQLException;
import java.util.*;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
public class ElasticSearchLogger {
private static Logger log = Logger.getLogger(ElasticSearchLogger.class);
private static boolean useProxies;
public static final String DATE_FORMAT_8601 = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'";
public static final String DATE_FORMAT_DCDATE = "yyyy-MM-dd'T'HH:mm:ss'Z'";
private static LookupService locationService;
public static String clusterName = "dspacestatslogging";
public static String indexName = "dspaceindex";
public static String indexType = "stats";
public static String address = "127.0.0.1";
public static int port = 9300;
private static Client client;
public static enum ClientType {
NODE, LOCAL, TRANSPORT
}
public ElasticSearchLogger() {
// nobody should be instantiating this...
}
public ElasticSearchLogger(boolean doInitialize) {
initializeElasticSearch();
}
public static ElasticSearchLogger getInstance() {
return ElasticSearchLoggerSingletonHolder.instance;
}
// Singleton Pattern of "Initialization on demand holder idiom"
private static class ElasticSearchLoggerSingletonHolder {
public static final ElasticSearchLogger instance = new ElasticSearchLogger(true);
}
public void initializeElasticSearch() {
log.info("DSpace ElasticSearchLogger Initializing");
try {
LookupService service = null;
// Get the db file for the location
String dbfile = ConfigurationManager.getProperty("usage-statistics", "dbfile");
if (dbfile != null) {
try {
service = new LookupService(dbfile, LookupService.GEOIP_STANDARD);
} catch (FileNotFoundException fe) {
log.error("The GeoLite Database file is missing (" + dbfile + ")! Usage Statistics cannot generate location based reports! Please see the DSpace installation instructions for instructions to install this file.", fe);
} catch (IOException e) {
log.error("Unable to load GeoLite Database file (" + dbfile + ")! You may need to reinstall it. See the DSpace installation instructions for more details.", e);
}
} else {
log.error("The required 'dbfile' configuration is missing in usage-statistics.cfg!");
}
locationService = service;
if ("true".equals(ConfigurationManager.getProperty("useProxies"))) {
useProxies = true;
} else {
useProxies = false;
}
log.info("useProxies=" + useProxies);
// Configurable values for all elasticsearch connection constants
clusterName = getConfigurationStringWithFallBack("elastic-search-statistics", "clusterName", clusterName);
indexName = getConfigurationStringWithFallBack("elastic-search-statistics", "indexName", indexName);
indexType = getConfigurationStringWithFallBack("elastic-search-statistics", "indexType", indexType);
address = getConfigurationStringWithFallBack("elastic-search-statistics", "address", address);
port = ConfigurationManager.getIntProperty("elastic-search-statistics", "port", port);
//Initialize the connection to Elastic Search, and ensure our index is available.
client = getClient();
IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest();
indicesExistsRequest.indices(new String[] {indexName});
ActionFuture<IndicesExistsResponse> actionFutureIndicesExist = client.admin().indices().exists(indicesExistsRequest);
log.info("DS ES Checking if index exists");
if(! actionFutureIndicesExist.actionGet().isExists() ) {
//If elastic search index exists, then we are good to go, otherwise, we need to create that index. Should only need to happen once ever.
log.info("DS ES index didn't exist, we need to create it.");
Settings settings = ImmutableSettings.settingsBuilder()
.put("number_of_replicas", 1)
.put("number_of_shards", 5)
.put("cluster.name", clusterName)
.build();
String stringMappingJSON = "{\""+indexType+"\" : { \"properties\" : {\n" +
" \"userAgent\":{\n" +
" \"type\":\"string\"\n" +
" },\n" +
" \"countryCode\":{\n" +
" \"type\":\"string\",\n" +
" \"index\":\"not_analyzed\",\n" +
" \"omit_norms\":true\n" +
" },\n" +
" \"dns\":{\n" +
" \"type\":\"multi_field\",\n" +
" \"fields\": {\n" +
" \"dns\": {\"type\":\"string\",\"index\":\"analyzed\"},\n" +
" \"untouched\":{\"type\":\"string\",\"index\":\"not_analyzed\"}\n" +
" }\n" +
" },\n" +
" \"isBot\":{\n" +
" \"type\":\"boolean\"\n" +
" },\n" +
" \"owningColl\":{\n" +
" \"type\":\"integer\",\n" +
" \"index\":\"not_analyzed\"\n" +
" },\n" +
" \"type\":{\n" +
" \"type\":\"string\",\n" +
" \"index\":\"not_analyzed\",\n" +
" \"omit_norms\":true\n" +
" },\n" +
" \"owningComm\":{\n" +
" \"type\":\"integer\",\n" +
" \"index\":\"not_analyzed\"\n" +
" },\n" +
" \"city\":{\n" +
" \"type\":\"multi_field\",\n" +
" \"fields\": {\n" +
" \"city\": {\"type\":\"string\",\"index\":\"analyzed\"},\n" +
" \"untouched\":{\"type\":\"string\",\"index\":\"not_analyzed\"}\n" +
" }\n" +
" },\n" +
" \"country\":{\n" +
" \"type\":\"multi_field\",\n" +
" \"fields\": {\n" +
" \"country\": {\"type\":\"string\",\"index\":\"analyzed\"},\n" +
" \"untouched\":{\"type\":\"string\",\"index\":\"not_analyzed\"}\n" +
" }\n" +
" },\n" +
" \"ip\":{\n" +
" \"type\":\"multi_field\",\n" +
" \"fields\": {\n" +
" \"ip\": {\"type\":\"string\",\"index\":\"analyzed\"},\n" +
" \"untouched\":{\"type\":\"string\",\"index\":\"not_analyzed\"}\n" +
" }\n" +
" },\n" +
" \"id\":{\n" +
" \"type\":\"integer\",\n" +
" \"index\":\"not_analyzed\"\n" +
" },\n" +
" \"time\":{\n" +
" \"type\":\"date\"\n" +
" },\n" +
" \"owningItem\":{\n" +
" \"type\":\"string\",\n" +
" \"index\":\"not_analyzed\"\n" +
" },\n" +
" \"continent\":{\n" +
" \"type\":\"string\",\n" +
" \"index\":\"not_analyzed\"\n" +
" },\n" +
" \"geo\":{\n" +
" \"type\":\"geo_point\"\n" +
" },\n" +
" \"bundleName\":{\n" +
" \"type\":\"string\",\n" +
" \"index\":\"not_analyzed\"\n" +
" },\n" +
" \"epersonid\":{\n" +
" \"type\":\"string\",\n" +
" \"index\":\"not_analyzed\"\n" +
" }\n" +
"} } }";
client.prepareIndex(indexName, indexType, "1")
.setSource(jsonBuilder()
.startObject()
.field("user", "kimchy")
.field("postDate", new Date())
.field("message", "trying out Elastic Search")
.endObject()
)
.execute()
.actionGet();
log.info("Create INDEX ["+indexName+"]/["+indexType+"]");
// Wait for create to be finished.
client.admin().indices().prepareRefresh(indexName).execute().actionGet();
//Put the schema/mapping
log.info("Put Mapping for ["+indexName+"]/["+indexType+"]="+stringMappingJSON);
PutMappingRequestBuilder putMappingRequestBuilder = client.admin().indices().preparePutMapping(indexName).setType(indexType);
putMappingRequestBuilder.setSource(stringMappingJSON);
PutMappingResponse response = putMappingRequestBuilder.execute().actionGet();
if(!response.getAcknowledged()) {
log.info("Could not define mapping for type ["+indexName+"]/["+indexType+"]");
} else {
log.info("Successfully put mapping for ["+indexName+"]/["+indexType+"]");
}
log.info("DS ES index didn't exist, but we created it.");
} else {
log.info("DS ES index already exists");
}
log.info("DSpace ElasticSearchLogger Initialized Successfully (I suppose)");
} catch (Exception e) {
log.info("Elastic Search crashed during init. " + e.getMessage());
}
}
public void post(DSpaceObject dspaceObject, HttpServletRequest request, EPerson currentUser) {
//log.info("DS-ES post for type:"+dspaceObject.getType() + " -- " + dspaceObject.getName());
client = ElasticSearchLogger.getInstance().getClient();
boolean isSpiderBot = SpiderDetector.isSpider(request);
try {
if (isSpiderBot &&
!ConfigurationManager.getBooleanProperty("usage-statistics", "logBots", true)) {
return;
}
// Save our basic info that we already have
String ip = request.getRemoteAddr();
if (isUseProxies() && request.getHeader("X-Forwarded-For") != null) {
/* This header is a comma delimited list */
for (String xfip : request.getHeader("X-Forwarded-For").split(",")) {
/* proxy itself will sometime populate this header with the same value in
remote address. ordering in spec is vague, we'll just take the last
not equal to the proxy
*/
if (!request.getHeader("X-Forwarded-For").contains(ip)) {
ip = xfip.trim();
}
}
}
XContentBuilder docBuilder = null;
docBuilder = XContentFactory.jsonBuilder().startObject();
docBuilder.field("ip", ip);
docBuilder.field("id", dspaceObject.getID());
// The numerical constant that represents the DSpaceObject TYPE. i.e. 0=bitstream, 2=item, ...
docBuilder.field("typeIndex", dspaceObject.getType());
// The text that represent the DSpaceObject TYPE. i.e. BITSTREAM, ITEM, COLLECTION, COMMUNITY
docBuilder.field("type", Constants.typeText[dspaceObject.getType()]);
// Save the current time
docBuilder.field("time", DateFormatUtils.format(new Date(), DATE_FORMAT_8601));
if (currentUser != null) {
docBuilder.field("epersonid", currentUser.getID());
}
try {
String dns = DnsLookup.reverseDns(ip);
docBuilder.field("dns", dns.toLowerCase());
} catch (Exception e) {
log.error("Failed DNS Lookup for IP:" + ip);
log.debug(e.getMessage(), e);
}
// Save the location information if valid, save the event without
// location information if not valid
Location location = locationService.getLocation(ip);
if (location != null
&& !("--".equals(location.countryCode)
&& location.latitude == -180 && location.longitude == -180)) {
try {
docBuilder.field("continent", LocationUtils
.getContinentCode(location.countryCode));
} catch (Exception e) {
System.out
.println("COUNTRY ERROR: " + location.countryCode);
}
docBuilder.field("countryCode", location.countryCode);
docBuilder.field("city", location.city);
docBuilder.field("latitude", location.latitude);
docBuilder.field("longitude", location.longitude);
docBuilder.field("isBot", isSpiderBot);
if (request.getHeader("User-Agent") != null) {
docBuilder.field("userAgent", request.getHeader("User-Agent"));
}
}
if (dspaceObject instanceof Bitstream) {
Bitstream bit = (Bitstream) dspaceObject;
Bundle[] bundles = bit.getBundles();
docBuilder.field("bundleName").startArray();
for (Bundle bundle : bundles) {
docBuilder.value(bundle.getName());
}
docBuilder.endArray();
}
storeParents(docBuilder, getParents(dspaceObject));
docBuilder.endObject();
if (docBuilder != null) {
IndexRequestBuilder irb = client.prepareIndex(indexName, indexType)
.setSource(docBuilder);
//log.info("Executing document insert into index");
if(client == null) {
log.error("Hey, client is null");
}
irb.execute().actionGet();
}
} catch (RuntimeException re) {
log.error("RunTimer in ESL:\n" + ExceptionUtils.getStackTrace(re));
throw re;
} catch (Exception e) {
log.error(e.getMessage());
} finally {
client.close();
}
}
public static String getClusterName() {
return clusterName;
}
public static void setClusterName(String clusterName) {
ElasticSearchLogger.clusterName = clusterName;
}
public static String getIndexName() {
return indexName;
}
public static void setIndexName(String indexName) {
ElasticSearchLogger.indexName = indexName;
}
public static String getIndexType() {
return indexType;
}
public static void setIndexType(String indexType) {
ElasticSearchLogger.indexType = indexType;
}
public static String getAddress() {
return address;
}
public static void setAddress(String address) {
ElasticSearchLogger.address = address;
}
public static int getPort() {
return port;
}
public static void setPort(int port) {
ElasticSearchLogger.port = port;
}
public void buildParents(DSpaceObject dso, HashMap<String, ArrayList<Integer>> parents)
throws SQLException {
if (dso instanceof Community) {
Community comm = (Community) dso;
while (comm != null && comm.getParentCommunity() != null) {
comm = comm.getParentCommunity();
parents.get("owningComm").add(comm.getID());
}
} else if (dso instanceof Collection) {
Collection coll = (Collection) dso;
for (Community community : coll.getCommunities()) {
parents.get("owningComm").add(community.getID());
buildParents(community, parents);
}
} else if (dso instanceof Item) {
Item item = (Item) dso;
for (Collection collection : item.getCollections()) {
parents.get("owningColl").add(collection.getID());
buildParents(collection, parents);
}
} else if (dso instanceof Bitstream) {
Bitstream bitstream = (Bitstream) dso;
for (Bundle bundle : bitstream.getBundles()) {
for (Item item : bundle.getItems()) {
parents.get("owningItem").add(item.getID());
buildParents(item, parents);
}
}
}
}
public HashMap<String, ArrayList<Integer>> getParents(DSpaceObject dso)
throws SQLException {
HashMap<String, ArrayList<Integer>> parents = new HashMap<String, ArrayList<Integer>>();
parents.put("owningComm", new ArrayList<Integer>());
parents.put("owningColl", new ArrayList<Integer>());
parents.put("owningItem", new ArrayList<Integer>());
buildParents(dso, parents);
return parents;
}
public void storeParents(XContentBuilder docBuilder, HashMap<String, ArrayList<Integer>> parents) throws IOException {
Iterator it = parents.keySet().iterator();
while (it.hasNext()) {
String key = (String) it.next();
ArrayList<Integer> ids = parents.get(key);
if (ids.size() > 0) {
docBuilder.field(key).startArray();
for (Integer i : ids) {
docBuilder.value(i);
}
docBuilder.endArray();
}
}
}
public boolean isUseProxies() {
return useProxies;
}
// Transport Client will talk to server on 9300
public void createTransportClient() {
// Configurable values for all elasticsearch connection constants
// Can't guarantee that these values are already loaded, since this can be called by a different JVM
clusterName = getConfigurationStringWithFallBack("elastic-search-statistics", "clusterName", clusterName);
indexName = getConfigurationStringWithFallBack("elastic-search-statistics", "indexName", indexName);
indexType = getConfigurationStringWithFallBack("elastic-search-statistics", "indexType", indexType);
address = getConfigurationStringWithFallBack("elastic-search-statistics", "address", address);
port = ConfigurationManager.getIntProperty("elastic-search-statistics", "port", port);
log.info("Creating TransportClient to [Address:" + address + "] [Port:" + port + "] [cluster.name:" + clusterName + "]");
Settings settings = ImmutableSettings.settingsBuilder().put("cluster.name", clusterName).build();
client = new TransportClient(settings).addTransportAddress(new InetSocketTransportAddress(address, port));
}
public Client getClient() {
//Get an available client, otherwise new default is NODE.
return getClient(ClientType.NODE);
}
// Get the already available client, otherwise we will create a new client.
// TODO Allow for config to determine which architecture / topology to use.
// - Local Node, store Data
// - Node Client, must discover a master within ES cluster
// - Transport Client, specify IP address of server running ES.
public Client getClient(ClientType clientType) {
if(client == null) {
log.error("getClient reports null client");
if(clientType == ClientType.TRANSPORT) {
createTransportClient();
} else {
createNodeClient(clientType);
}
}
return client;
}
// Node Client will discover other ES nodes running in local JVM
public Client createNodeClient(ClientType clientType) {
String dspaceDir = ConfigurationManager.getProperty("dspace.dir");
Settings settings = ImmutableSettings.settingsBuilder().put("path.data", dspaceDir + "/elasticsearch/").build();
NodeBuilder nodeBuilder = NodeBuilder.nodeBuilder().clusterName(clusterName).data(true).settings(settings);
if(clientType == ClientType.LOCAL) {
log.info("Create a Local Node.");
nodeBuilder = nodeBuilder.local(true);
} else if(clientType == ClientType.NODE) {
log.info("Create a nodeClient, allows transport clients to connect");
nodeBuilder = nodeBuilder.local(false);
}
Node node = nodeBuilder.node();
log.info("Got node");
client = node.client();
log.info("Created new node client");
return client;
}
public String getConfigurationStringWithFallBack(String module, String configurationKey, String defaultFallbackValue) {
String configDrivenValue = ConfigurationManager.getProperty(module, configurationKey);
if(configDrivenValue == null || configDrivenValue.trim().equalsIgnoreCase("")) {
return defaultFallbackValue;
} else {
return configDrivenValue;
}
}
}

40
dspace-stats/src/main/java/org/dspace/statistics/ElasticSearchLoggerEventListener.java Normal file
View File

@@ -0,0 +1,40 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.statistics;
import org.apache.log4j.Logger;
import org.dspace.eperson.EPerson;
import org.dspace.services.model.Event;
import org.dspace.usage.AbstractUsageEventListener;
import org.dspace.usage.UsageEvent;
public class ElasticSearchLoggerEventListener extends AbstractUsageEventListener {
private static Logger log = Logger.getLogger(ElasticSearchLoggerEventListener.class);
public void receiveEvent(Event event) {
if(event instanceof UsageEvent && (((UsageEvent) event).getAction() == UsageEvent.Action.VIEW))
{
try{
UsageEvent ue = (UsageEvent) event;
EPerson currentUser = ue.getContext() == null ? null : ue.getContext().getCurrentUser();
ElasticSearchLogger.getInstance().post(ue.getObject(), ue.getRequest(), currentUser);
log.info("Successfully logged " + ue.getObject().getTypeText() + "_" + ue.getObject().getID() + " " + ue.getObject().getName());
}
catch(Exception e)
{
log.error("General Exception: " + e.getMessage());
}
}
}
}

9
dspace-stats/src/main/java/org/dspace/statistics/SolrLogger.java
View File

@@ -77,7 +77,7 @@ public class SolrLogger
{
log.info("solr-statistics.spidersfile:" + ConfigurationManager.getProperty("solr-statistics", "spidersfile"));
log.info("solr-statistics.server:" + ConfigurationManager.getProperty("solr-statistics", "server"));
log.info("solr-statistics.dbfile:" + ConfigurationManager.getProperty("solr-statistics", "dbfile"));
log.info("usage-statistics.dbfile:" + ConfigurationManager.getProperty("usage-statistics", "dbfile"));
CommonsHttpSolrServer server = null;
@@ -100,7 +100,7 @@ public class SolrLogger
LookupService service = null;
// Get the db file for the location
String dbfile = ConfigurationManager.getProperty("solr-statistics", "dbfile");
String dbfile = ConfigurationManager.getProperty("usage-statistics", "dbfile");
if (dbfile != null)
{
try
@@ -119,7 +119,7 @@ public class SolrLogger
}
else
{
log.error("The required 'dbfile' configuration is missing in solr-statistics.cfg!");
log.error("The required 'dbfile' configuration is missing in usage-statistics.cfg!");
}
locationService = service;
@@ -168,7 +168,7 @@ public class SolrLogger
try
{
if(isSpiderBot &&
!ConfigurationManager.getBooleanProperty("solr-statistics", "logBots",true))
!ConfigurationManager.getBooleanProperty("usage-statistics", "logBots",true))
{
return;
}
@@ -295,6 +295,7 @@ public class SolrLogger
}
}
//@TODO remove metadataStorage object from all see: DS-421
public static Map<String, String> getMetadataStorageInfo()
{
return metadataStorageInfo;

2
dspace-stats/src/main/java/org/dspace/statistics/util/DnsLookup.java
View File

@@ -24,7 +24,7 @@ public class DnsLookup {
Resolver res = new ExtendedResolver();
// set the timeout, defaults to 200 milliseconds
int timeout = ConfigurationManager.getIntProperty("solr-statistics", "resolver.timeout", 200);
int timeout = ConfigurationManager.getIntProperty("usage-statistics", "resolver.timeout", 200);
res.setTimeout(0, timeout);
Name name = ReverseMap.fromAddress(hostIp);

21
dspace-stats/src/main/java/org/dspace/statistics/util/SpiderDetector.java
View File

@@ -9,7 +9,6 @@ package org.dspace.statistics.util;
import org.apache.log4j.Logger;
import org.dspace.core.ConfigurationManager;
import org.dspace.statistics.SolrLogger;
import javax.servlet.http.HttpServletRequest;
import java.io.BufferedReader;
@@ -32,6 +31,8 @@ public class SpiderDetector {
private static Logger log = Logger.getLogger(SpiderDetector.class);
private static Boolean useProxies;
/**
* Sparse HashTable structure to hold IP address ranges.
*/
@@ -138,7 +139,7 @@ public class SpiderDetector {
*/
public static boolean isSpider(HttpServletRequest request) {
if (SolrLogger.isUseProxies() && request.getHeader("X-Forwarded-For") != null) {
if (isUseProxies() && request.getHeader("X-Forwarded-For") != null) {
/* This header is a comma delimited list */
for (String xfip : request.getHeader("X-Forwarded-For").split(",")) {
if (isSpider(xfip))
@@ -177,4 +178,20 @@ public class SpiderDetector {
}
private static boolean isUseProxies() {
if(useProxies == null) {
if ("true".equals(ConfigurationManager.getProperty("useProxies")))
{
useProxies = true;
}
else
{
useProxies = false;
}
}
return useProxies;
}
}

2
dspace-stats/src/main/java/org/dspace/statistics/util/StatisticsDataGenerator.java
View File

@@ -199,7 +199,7 @@ public class StatisticsDataGenerator {
Map<String, String> metadataStorageInfo = SolrLogger.getMetadataStorageInfo();
String prevIp = null;
String dbfile = ConfigurationManager.getProperty("solr-statistics", "dbfile");
String dbfile = ConfigurationManager.getProperty("usage-statistics", "dbfile");
LookupService cl = new LookupService(dbfile,
LookupService.GEOIP_STANDARD);
int countryErrors = 0;

2
dspace-stats/src/main/java/org/dspace/statistics/util/StatisticsImporter.java
View File

@@ -470,7 +470,7 @@ public class StatisticsImporter
solr = new CommonsHttpSolrServer(sserver);
metadataStorageInfo = SolrLogger.getMetadataStorageInfo();
String dbfile = ConfigurationManager.getProperty("solr-statistics", "dbfile");
String dbfile = ConfigurationManager.getProperty("usage-statistics", "dbfile");
try
{
geoipLookup = new LookupService(dbfile, LookupService.GEOIP_STANDARD);

450
dspace-stats/src/main/java/org/dspace/statistics/util/StatisticsImporterElasticSearch.java Normal file
View File

@@ -0,0 +1,450 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.statistics.util;
import com.maxmind.geoip.Location;
import com.maxmind.geoip.LookupService;
import org.apache.commons.cli.*;
import org.apache.commons.lang.time.DateFormatUtils;
import org.apache.log4j.Logger;
import org.dspace.content.Bitstream;
import org.dspace.content.Bundle;
import org.dspace.content.DSpaceObject;
import org.dspace.core.ConfigurationManager;
import org.dspace.core.Constants;
import org.dspace.core.Context;
import org.dspace.eperson.EPerson;
import org.dspace.statistics.ElasticSearchLogger;
import org.dspace.statistics.SolrLogger;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.action.bulk.BulkRequestBuilder;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.mapper.geo.GeoPoint;
import java.io.*;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Random;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
/**
* Created by IntelliJ IDEA.
* User: peterdietz
* Date: 8/15/12
* Time: 2:46 PM
* To change this template use File | Settings | File Templates.
*/
public class StatisticsImporterElasticSearch {
private static final Logger log = Logger.getLogger(StatisticsImporterElasticSearch.class);
/** Date format */
private static SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
//TODO ES Client
/** GEOIP lookup service */
private static LookupService geoipLookup;
/** Metadata storage information */
private static Map<String, String> metadataStorageInfo;
/** Whether to skip the DNS reverse lookup or not */
private static boolean skipReverseDNS = false;
private static ElasticSearchLogger elasticSearchLoggerInstance;
private static Client client;
private static BulkRequestBuilder bulkRequest;
/**
* Read lines from the statistics file and load their data into Elastic Search.
*
* @param filename The filename of the file to load
* @param context The DSpace Context
* @param verbose Whether to display verbose output
*/
private void load(String filename, Context context, boolean verbose)
{
// Item counter
int counter = 0;
int errors = 0;
int searchengines = 0;
try
{
BufferedReader input;
if (null == filename || "-".equals(filename))
{
input = new BufferedReader(new InputStreamReader(System.in));
filename = "standard input";
}
else
input = new BufferedReader(new FileReader(new File(filename)));
// Print out the filename for confirmation
System.out.println("Processing file: " + filename);
String line;
// String uuid;
String action;
String id;
Date date;
String user;
String ip;
String continent = "";
String country = "";
String countryCode = "";
float longitude = 0f;
float latitude = 0f;
String city = "";
String dns;
DNSCache dnsCache = new DNSCache(2500, 0.75f, 2500);
Object fromCache;
Random rand = new Random();
while ((line = input.readLine()) != null)
{
// Tokenise the line
counter++;
errors++;
if (verbose)
{
System.out.println("Line:" + line);
}
String[] parts = line.split(",");
// uuid = parts[0];
action = parts[1];
id = parts[2];
date = dateFormat.parse(parts[3]);
user = parts[4];
ip = parts[5];
// Resolve the dns (if applicable) to get rid of search engine bots early on in the processing chain
dns = "";
if (!skipReverseDNS)
{
// Is the IP address in the cache?
fromCache = dnsCache.get(ip);
if (fromCache != null)
{
dns = (String)fromCache;
}
else
{
try
{
dns = DnsLookup.reverseDns(ip);
dnsCache.put(ip, dns);
} catch (Exception e)
{
dns = "";
}
}
}
String data = "";
data += ("ip addr = " + ip);
data += (", dns name = " + dns);
if ((dns.endsWith(".googlebot.com.")) ||
(dns.endsWith(".crawl.yahoo.net.")) ||
(dns.endsWith(".search.msn.com.")))
{
if (verbose)
{
System.out.println(data + ", IGNORE (search engine)");
}
errors--;
searchengines++;
continue;
}
// Get the geo information for the user
Location location;
try {
location = geoipLookup.getLocation(ip);
city = location.city;
country = location.countryName;
countryCode = location.countryCode;
longitude = location.longitude;
latitude = location.latitude;
if(verbose) {
data += (", country = " + country);
data += (", city = " + city);
System.out.println(data);
}
try {
continent = LocationUtils.getContinentCode(countryCode);
} catch (Exception e) {
if (verbose)
{
System.out.println("Unknown country code: " + countryCode);
}
continue;
}
} catch (Exception e) {
// No problem - just can't look them up
}
// Now find our dso
int type = 0;
if ("view_bitstream".equals(action))
{
type = Constants.BITSTREAM;
}
else if ("view_item".equals(action))
{
type = Constants.ITEM;
}
else if ("view_collection".equals(action))
{
type = Constants.COLLECTION;
}
else if ("view_community".equals(action))
{
type = Constants.COMMUNITY;
}
DSpaceObject dso = DSpaceObject.find(context, type, Integer.parseInt(id));
if (dso == null)
{
if (verbose)
{
System.err.println(" - DSO with ID '" + id + "' is no longer in the system");
}
continue;
}
// Get the eperson details
EPerson eperson = EPerson.findByEmail(context, user);
int epersonId = 0;
if (eperson != null)
{
eperson.getID();
}
//TODO Is there any way to reuse ElasticSearchLogger.post() ?
// Save it in our server
XContentBuilder postBuilder = jsonBuilder().startObject()
.field("id", dso.getID())
.field("typeIndex", dso.getType())
.field("type", dso.getTypeText())
.field("geo", new GeoPoint(latitude, longitude))
.field("continent", continent)
.field("countryCode", countryCode)
.field("country", country)
.field("city", city)
.field("ip", ip)
.field("time", DateFormatUtils.format(date, SolrLogger.DATE_FORMAT_8601));
// Unable to get UserAgent from logs. .field("userAgent")
if (dso instanceof Bitstream) {
Bitstream bit = (Bitstream) dso;
Bundle[] bundles = bit.getBundles();
postBuilder = postBuilder.field("bundleName").startArray();
for (Bundle bundle : bundles) {
postBuilder = postBuilder.value(bundle.getName());
}
postBuilder = postBuilder.endArray();
}
if (epersonId > 0)
{
postBuilder = postBuilder.field("epersonid", epersonId);
}
if (dns != null)
{
postBuilder = postBuilder.field("dns", dns.toLowerCase());
}
//Save for later: .field("isBot")
elasticSearchLoggerInstance.storeParents(postBuilder, elasticSearchLoggerInstance.getParents(dso));
bulkRequest.add(client.prepareIndex(elasticSearchLoggerInstance.getIndexName(), elasticSearchLoggerInstance.getIndexType())
.setSource(postBuilder.endObject()));
errors--;
}
if(bulkRequest.numberOfActions() > 0) {
BulkResponse bulkResponse = bulkRequest.execute().actionGet();
if(bulkResponse.hasFailures()) {
log.error("Bulk Request Failed due to: " + bulkResponse.buildFailureMessage());
}
}
}
catch (RuntimeException re)
{
throw re;
}
catch (Exception e)
{
System.err.println(e.getMessage());
log.error(e.getMessage(), e);
}
DecimalFormat percentage = new DecimalFormat("##.###");
int committed = counter - errors - searchengines;
System.out.println("Processed " + counter + " log lines");
if (counter > 0)
{
Double committedpercentage = 100d * committed / counter;
System.out.println(" - " + committed + " entries added to ElasticSearch: " + percentage.format(committedpercentage) + "%");
Double errorpercentage = 100d * errors / counter;
System.out.println(" - " + errors + " errors: " + percentage.format(errorpercentage) + "%");
Double sepercentage = 100d * searchengines / counter;
System.out.println(" - " + searchengines + " search engine activity skipped: " + percentage.format(sepercentage) + "%");
}
System.out.println(" done!");
}
/**
* Print the help message
*
* @param options The command line options the user gave
* @param exitCode the system exit code to use
*/
private static void printHelp(Options options, int exitCode)
{
// print the help message
HelpFormatter myhelp = new HelpFormatter();
myhelp.printHelp("StatisticsImporterElasticSearch\n", options);
System.exit(exitCode);
}
/**
* Main method to run the statistics importer.
*
* @param args The command line arguments
* @throws Exception If something goes wrong
*/
public static void main(String[] args) throws Exception
{
CommandLineParser parser = new PosixParser();
Options options = new Options();
options.addOption("i", "in", true, "the input file ('-' or omit for standard input)");
options.addOption("m", "multiple", false, "treat the input file as having a wildcard ending");
options.addOption("s", "skipdns", false, "skip performing reverse DNS lookups on IP addresses");
options.addOption("v", "verbose", false, "display verbose output (useful for debugging)");
options.addOption("h", "help", false, "help");
CommandLine line = parser.parse(options, args);
// Did the user ask to see the help?
if (line.hasOption('h'))
{
printHelp(options, 0);
}
if (line.hasOption('s'))
{
skipReverseDNS = true;
}
elasticSearchLoggerInstance = new ElasticSearchLogger();
log.info("Getting ElasticSearch Transport Client for StatisticsImporterElasticSearch...");
// This is only invoked via terminal, do not use _this_ node as that data storing node.
// Need to get a NodeClient or TransportClient, but definitely do not want to get a local data storing client.
client = elasticSearchLoggerInstance.getClient(ElasticSearchLogger.ClientType.TRANSPORT);
client.admin().indices().prepareRefresh(ElasticSearchLogger.getIndexName()).execute().actionGet();
bulkRequest = client.prepareBulk();
// We got all our parameters now get the rest
Context context = new Context();
// Verbose option
boolean verbose = line.hasOption('v');
String dbfile = ConfigurationManager.getProperty("usage-statistics", "dbfile");
try
{
geoipLookup = new LookupService(dbfile, LookupService.GEOIP_STANDARD);
}
catch (FileNotFoundException fe)
{
log.error("The GeoLite Database file is missing (" + dbfile + ")! Elastic Search Statistics cannot generate location based reports! Please see the DSpace installation instructions for instructions to install this file.", fe);
}
catch (IOException e)
{
log.error("Unable to load GeoLite Database file (" + dbfile + ")! You may need to reinstall it. See the DSpace installation instructions for more details.", e);
}
StatisticsImporterElasticSearch elasticSearchImporter = new StatisticsImporterElasticSearch();
if (line.hasOption('m'))
{
// Convert all the files
final File sample = new File(line.getOptionValue('i'));
File dir = sample.getParentFile();
FilenameFilter filter = new FilenameFilter()
{
public boolean accept(File dir, String name)
{
return name.startsWith(sample.getName());
}
};
String[] children = dir.list(filter);
for (String in : children)
{
System.out.println(in);
elasticSearchImporter.load(dir.getAbsolutePath() + System.getProperty("file.separator") + in, context, verbose);
}
}
else
{
// Just convert the one file
elasticSearchImporter.load(line.getOptionValue('i'), context, verbose);
}
}
/**
* Inner class to hold a cache of reverse lookups of IP addresses
* @param <K>
* @param <V>
*/
static class DNSCache<K,V> extends LinkedHashMap<K,V>
{
private int maxCapacity;
public DNSCache(int initialCapacity, float loadFactor, int maxCapacity)
{
super(initialCapacity, loadFactor, true);
this.maxCapacity = maxCapacity;
}
@Override
protected boolean removeEldestEntry(java.util.Map.Entry<K,V> eldest)
{
return size() >= this.maxCapacity;
}
}
}

18
dspace-xmlui/dspace-xmlui-api/pom.xml
View File

@@ -58,6 +58,24 @@
<artifactId>yuicompressor</artifactId>
<version>2.3.6</version>
</dependency>
<!-- Needed for Form Validation -->
<dependency>
<groupId>commons-validator</groupId>
<artifactId>commons-validator</artifactId>
<version>1.3.1</version>
</dependency>
<dependency>
<groupId>net.sf.opencsv</groupId>
<artifactId>opencsv</artifactId>
<version>2.0</version>
</dependency>
<!-- Gson: Java to Json conversion -->
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.1</version>
<scope>compile</scope>
</dependency>
</dependencies>

2
dspace-xmlui/dspace-xmlui-api/src/main/java/org/dspace/app/xmlui/aspect/statistics/StatisticsAuthorizedMatcher.java
View File

@@ -56,7 +56,7 @@ public class StatisticsAuthorizedMatcher extends AbstractLogEnabled implements M
//We have always got rights to view stats on the home page (admin rights will be checked later)
boolean authorized = dso == null || AuthorizeManager.authorizeActionBoolean(context, dso, action, false);
//If we are authorized check for any other authorization actions present
if(authorized && ConfigurationManager.getBooleanProperty("solr-statistics", "authorization.admin"))
if(authorized && ConfigurationManager.getBooleanProperty("usage-statistics", "authorization.admin"))
{
//If we have no user, we cannot be admin
if(context.getCurrentUser() == null)

45
dspace-xmlui/dspace-xmlui-api/src/main/java/org/dspace/app/xmlui/aspect/statistics/StatisticsTransformer.java
View File

@@ -7,25 +7,35 @@
*/
package org.dspace.app.xmlui.aspect.statistics;
import java.io.IOException;
import java.sql.SQLException;
import java.text.ParseException;
import com.google.gson.Gson;
import org.apache.log4j.Logger;
import org.apache.solr.client.solrj.SolrServerException;
import org.dspace.app.xmlui.cocoon.AbstractDSpaceTransformer;
import org.dspace.app.xmlui.utils.HandleUtil;
import org.dspace.app.xmlui.utils.UIException;
import org.dspace.app.xmlui.wing.WingException;
import org.dspace.app.xmlui.wing.Message;
import org.dspace.app.xmlui.wing.WingException;
import org.dspace.app.xmlui.wing.element.*;
import org.dspace.authorize.AuthorizeException;
import org.dspace.content.DSpaceObject;
import org.dspace.core.Constants;
import org.dspace.core.Context;
import org.dspace.statistics.Dataset;
import org.dspace.statistics.ObjectCount;
import org.dspace.statistics.content.*;
import org.dspace.storage.rdbms.DatabaseManager;
import org.dspace.storage.rdbms.TableRow;
import org.dspace.storage.rdbms.TableRowIterator;
import org.xml.sax.SAXException;
import java.io.IOException;
import java.sql.SQLException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.Locale;
public class StatisticsTransformer extends AbstractDSpaceTransformer {
private static Logger log = Logger.getLogger(StatisticsTransformer.class);
@@ -40,6 +50,30 @@ public class StatisticsTransformer extends AbstractDSpaceTransformer {
private static final String T_head_visits_cities = "xmlui.statistics.visits.cities";
private static final String T_head_visits_bitstream = "xmlui.statistics.visits.bitstreams";
private Date dateStart = null;
private Date dateEnd = null;
private SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
public StatisticsTransformer(Date dateStart, Date dateEnd) {
this.dateStart = dateStart;
this.dateEnd = dateEnd;
try {
this.context = new Context();
} catch (SQLException e) {
log.error("Error getting context in StatisticsTransformer:" + e.getMessage());
}
}
public StatisticsTransformer() {
try {
this.context = new Context();
} catch (SQLException e) {
log.error("Error getting context in StatisticsTransformer:" + e.getMessage());
}
}
/**
* Add a page title and trail links
*/
@@ -387,4 +421,5 @@ public class StatisticsTransformer extends AbstractDSpaceTransformer {
}
}
}

242
dspace-xmlui/dspace-xmlui-api/src/main/java/org/dspace/app/xmlui/aspect/statisticsElasticSearch/CSVOutputter.java Normal file
View File

@@ -0,0 +1,242 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.xmlui.aspect.statisticsElasticSearch;
import au.com.bytecode.opencsv.CSVWriter;
import org.apache.avalon.excalibur.pool.Recyclable;
import org.apache.avalon.framework.parameters.Parameters;
import org.apache.cocoon.ProcessingException;
import org.apache.cocoon.environment.ObjectModelHelper;
import org.apache.cocoon.environment.Request;
import org.apache.cocoon.environment.Response;
import org.apache.cocoon.environment.SourceResolver;
import org.apache.cocoon.reading.AbstractReader;
import org.apache.log4j.Logger;
import org.dspace.app.xmlui.aspect.statistics.StatisticsTransformer;
import org.dspace.app.xmlui.utils.ContextUtil;
import org.dspace.app.xmlui.utils.HandleUtil;
import org.dspace.app.xmlui.wing.WingException;
import org.dspace.content.Bitstream;
import org.dspace.content.DCValue;
import org.dspace.content.DSpaceObject;
import org.dspace.content.Item;
import org.dspace.core.Context;
import org.dspace.storage.rdbms.TableRow;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.action.search.SearchRequestBuilder;
import org.elasticsearch.search.facet.datehistogram.DateHistogramFacet;
import org.elasticsearch.search.facet.terms.TermsFacet;
import org.xml.sax.SAXException;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.sql.SQLException;
import java.text.SimpleDateFormat;
import java.util.*;
/**
* Provides the usage statistics in CSV form
* @author Peter Dietz (pdietz84@gmail.com)
*/
public class CSVOutputter extends AbstractReader implements Recyclable
{
protected static final Logger log = Logger.getLogger(CSVOutputter.class);
private static SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
protected Response response;
protected Request request;
protected Context context;
protected CSVWriter writer = null;
public void setup(SourceResolver sourceResolver, Map objectModel, String src, Parameters parameters) throws IOException, SAXException, ProcessingException {
log.info("CSV Writer for stats");
super.setup(sourceResolver, objectModel, src, parameters);
try {
//super.setup(sourceResolver, objectModel, src, parameters);
this.request = ObjectModelHelper.getRequest(objectModel);
this.response = ObjectModelHelper.getResponse(objectModel);
context = ContextUtil.obtainContext(objectModel);
String requestURI = request.getRequestURI();
String[] uriSegments = requestURI.split("/");
String requestedReport = uriSegments[uriSegments.length-1];
if(requestedReport == null || requestedReport.length() < 1) {
response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
return;
}
response.setContentType("text/csv; encoding='UTF-8'");
response.setStatus(HttpServletResponse.SC_OK);
writer = new CSVWriter(response.getWriter());
DSpaceObject dso = HandleUtil.obtainHandle(objectModel);
response.setHeader("Content-Disposition", "attachment; filename=KBStats-" + dso.getHandle() + "-" + requestedReport +".csv");
Map<String, String> params = new HashMap<String, String>();
for (Enumeration<String> paramNames = (Enumeration<String>) request.getParameterNames(); paramNames.hasMoreElements(); ) {
String param = paramNames.nextElement();
params.put(param, request.getParameter(param));
}
String fromValue = "";
if(params.containsKey("from")) {
fromValue = params.get("from");
}
String toValue = "";
if(params.containsKey("to")) {
toValue = params.get("to");
}
Date fromDate;
ReportGenerator reportGeneratorInstance = new ReportGenerator();
if(fromValue.length() > 0) {
fromDate = reportGeneratorInstance.tryParse(fromValue);
} else {
fromDate = null;
}
Date toDate;
if(toValue.length() > 0) {
toDate = reportGeneratorInstance.tryParse(toValue);
} else {
toDate = null;
}
ElasticSearchStatsViewer esStatsViewer = new ElasticSearchStatsViewer(dso, fromDate, toDate);
StatisticsTransformer statisticsTransformerInstance = new StatisticsTransformer(fromDate, toDate);
if(requestedReport.equalsIgnoreCase("topCountries"))
{
SearchRequestBuilder requestBuilder = esStatsViewer.facetedQueryBuilder(esStatsViewer.facetTopCountries);
SearchResponse searchResponse = requestBuilder.execute().actionGet();
TermsFacet topCountriesFacet = searchResponse.getFacets().facet(TermsFacet.class, "top_countries");
addTermFacetToWriter(topCountriesFacet, "");
}
else if (requestedReport.equalsIgnoreCase("fileDownloads"))
{
SearchRequestBuilder requestBuilder = esStatsViewer.facetedQueryBuilder(esStatsViewer.facetMonthlyDownloads);
SearchResponse searchResponse = requestBuilder.execute().actionGet();
DateHistogramFacet monthlyDownloadsFacet = searchResponse.getFacets().facet(DateHistogramFacet.class, "monthly_downloads");
addDateHistogramFacetToWriter(monthlyDownloadsFacet);
}
else if(requestedReport.equalsIgnoreCase("topDownloads"))
{
SearchRequestBuilder requestBuilder = esStatsViewer.facetedQueryBuilder(esStatsViewer.facetTopBitstreamsAllTime);
SearchResponse searchResponse = requestBuilder.execute().actionGet();
log.info(searchResponse.toString());
TermsFacet topBitstreams = searchResponse.getFacets().facet(TermsFacet.class, "top_bitstreams_alltime");
addTermFacetToWriter(topBitstreams, "bitstream");
}
else {
response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
}
} catch (SQLException e) {
log.error("Some Error:" + e.getMessage());
} catch (WingException e) {
log.error("Some Error:" + e.getMessage());
} catch (IOException e) {
log.error("Some Error:" + e.getMessage());
} finally {
try {
if(writer != null) {
writer.close();
} else {
log.error("CSV Writer was null!!");
}
} catch (IOException e) {
log.error("Hilarity Ensues... IO Exception while closing the csv writer.");
}
}
}
private void addTermFacetToWriter(TermsFacet termsFacet, String termType) throws SQLException {
List<? extends TermsFacet.Entry> termsFacetEntries = termsFacet.getEntries();
if(termType.equalsIgnoreCase("bitstream")) {
writer.writeNext(new String[]{"BitstreamID", "Bitstream Name", "Bitstream Bundle", "Item Title", "Item Handle", "Item Creator", "Item Publisher", "Item Issue Date", "Count"});
} else {
writer.writeNext(new String[]{"term", "count"});
}
if(termsFacetEntries.size() == 0) {
return;
}
for(TermsFacet.Entry facetEntry : termsFacetEntries)
{
if(termType.equalsIgnoreCase("bitstream"))
{
Bitstream bitstream = Bitstream.find(context, Integer.parseInt(facetEntry.getTerm()));
Item item = (Item) bitstream.getParentObject();
String[] entryValues = new String[9];
entryValues[0] = bitstream.getID() + "";
entryValues[1] = bitstream.getName();
entryValues[2] = bitstream.getBundles()[0].getName();
entryValues[3] = item.getName();
entryValues[4] = "http://hdl.handle.net/" + item.getHandle();
entryValues[5] = wrapInDelimitedString(item.getMetadata("dc.creator"));
entryValues[6] = wrapInDelimitedString(item.getMetadata("dc.publisher"));
entryValues[7] = wrapInDelimitedString(item.getMetadata("dc.date.issued"));
entryValues[8] = facetEntry.getCount() + "";
writer.writeNext(entryValues);
} else {
writer.writeNext(new String[]{facetEntry.getTerm(), String.valueOf(facetEntry.getCount())});
}
}
}
public String wrapInDelimitedString(DCValue[] metadataEntries) {
StringBuilder metadataString = new StringBuilder();
for(DCValue metadataEntry : metadataEntries) {
if(metadataString.length() > 0) {
// Delimit entries with the || double pipe character sequence.
metadataString.append("\\|\\|");
}
metadataString.append(metadataEntry.value);
}
return metadataString.toString();
}
private void addDateHistogramFacetToWriter(DateHistogramFacet dateHistogramFacet) {
List<? extends DateHistogramFacet.Entry> monthlyFacetEntries = dateHistogramFacet.getEntries();
if(monthlyFacetEntries.size() == 0) {
return;
}
writer.writeNext(new String[]{"Month", "Count"});
for(DateHistogramFacet.Entry histogramEntry : monthlyFacetEntries) {
Date facetDate = new Date(histogramEntry.getTime());
writer.writeNext(new String[]{dateFormat.format(facetDate), String.valueOf(histogramEntry.getCount())});
}
}
public void generate() throws IOException {
log.info("CSV Writer generator for stats");
out.flush();
out.close();
}
public void recycle() {
this.request = null;
this.response = null;
}
}

418
dspace-xmlui/dspace-xmlui-api/src/main/java/org/dspace/app/xmlui/aspect/statisticsElasticSearch/ElasticSearchStatsViewer.java Normal file
View File

@@ -0,0 +1,418 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.xmlui.aspect.statisticsElasticSearch;
import org.apache.cocoon.environment.ObjectModelHelper;
import org.apache.cocoon.environment.Request;
import org.apache.log4j.Logger;
import org.dspace.app.xmlui.aspect.statistics.StatisticsTransformer;
import org.dspace.app.xmlui.cocoon.AbstractDSpaceTransformer;
import org.dspace.app.xmlui.utils.HandleUtil;
import org.dspace.app.xmlui.wing.Message;
import org.dspace.app.xmlui.wing.WingException;
import org.dspace.app.xmlui.wing.element.*;
import org.dspace.content.*;
import org.dspace.content.Item;
import org.dspace.core.Constants;
import org.dspace.statistics.DataTermsFacet;
import org.dspace.statistics.ElasticSearchLogger;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.action.search.SearchRequestBuilder;
import org.elasticsearch.index.query.*;
import org.elasticsearch.search.facet.AbstractFacetBuilder;
import org.elasticsearch.search.facet.FacetBuilders;
import org.elasticsearch.search.facet.datehistogram.DateHistogramFacet;
import org.elasticsearch.search.facet.terms.TermsFacet;
import java.sql.SQLException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.List;
/**
* Usage Statistics viewer, powered by Elastic Search.
* Allows for the user to dig deeper into the statistics for topDownloads, topCountries, etc.
* @author Peter Dietz (pdietz84@gmail.com)
*/
public class ElasticSearchStatsViewer extends AbstractDSpaceTransformer {
private static Logger log = Logger.getLogger(ElasticSearchStatsViewer.class);
public static final String elasticStatisticsPath = "stats";
private static SimpleDateFormat monthAndYearFormat = new SimpleDateFormat("MMMMM yyyy");
private static SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
private static Client client;
private static Division division;
private static DSpaceObject dso;
private static Date dateStart;
private static Date dateEnd;
protected static TermFilterBuilder justOriginals = FilterBuilders.termFilter("bundleName", "ORIGINAL");
protected static AbstractFacetBuilder facetTopCountries = FacetBuilders.termsFacet("top_countries").field("country.untouched").size(150)
.facetFilter(FilterBuilders.andFilter(
justOriginals,
FilterBuilders.notFilter(FilterBuilders.termFilter("country.untouched", "")))
);
protected static AbstractFacetBuilder facetMonthlyDownloads = FacetBuilders.dateHistogramFacet("monthly_downloads").field("time").interval("month")
.facetFilter(FilterBuilders.andFilter(
FilterBuilders.termFilter("type", "BITSTREAM"),
justOriginals
));
protected static AbstractFacetBuilder facetTopBitstreamsAllTime = FacetBuilders.termsFacet("top_bitstreams_alltime").field("id")
.facetFilter(FilterBuilders.andFilter(
FilterBuilders.termFilter("type", "BITSTREAM"),
justOriginals
));
protected static AbstractFacetBuilder facetTopUSCities = FacetBuilders.termsFacet("top_US_cities").field("city.untouched").size(50)
.facetFilter(FilterBuilders.andFilter(
FilterBuilders.termFilter("countryCode", "US"),
justOriginals,
FilterBuilders.notFilter(FilterBuilders.termFilter("city.untouched", ""))
));
protected static AbstractFacetBuilder facetTopUniqueIP = FacetBuilders.termsFacet("top_unique_ips").field("ip");
protected static AbstractFacetBuilder facetTopTypes = FacetBuilders.termsFacet("top_types").field("type");
/** Language strings */
private static final Message T_dspace_home = message("xmlui.general.dspace_home");
private static final Message T_trail = message("xmlui.ArtifactBrowser.ItemViewer.trail");
public void addPageMeta(PageMeta pageMeta) throws WingException, SQLException {
DSpaceObject dso = HandleUtil.obtainHandle(objectModel);
pageMeta.addMetadata("title").addContent("Statistics Report for : " + dso.getName());
pageMeta.addTrailLink(contextPath + "/",T_dspace_home);
HandleUtil.buildHandleTrail(dso,pageMeta,contextPath, true);
pageMeta.addTrail().addContent("View Statistics");
}
public ElasticSearchStatsViewer() {
}
public ElasticSearchStatsViewer(DSpaceObject dso, Date dateStart, Date dateEnd) {
this.dso = dso;
this.dateStart = dateStart;
this.dateEnd = dateEnd;
client = ElasticSearchLogger.getInstance().getClient();
}
public void addBody(Body body) throws WingException, SQLException {
try {
//Try to find our dspace object
dso = HandleUtil.obtainHandle(objectModel);
client = ElasticSearchLogger.getInstance().getClient();
division = body.addDivision("elastic-stats");
division.setHead("Statistical Report for " + dso.getName());
division.addHidden("containerName").setValue(dso.getName());
division.addHidden("baseURLStats").setValue(contextPath + "/handle/" + dso.getHandle() + "/" + elasticStatisticsPath);
Request request = ObjectModelHelper.getRequest(objectModel);
String[] requestURIElements = request.getRequestURI().split("/");
// If we are on the homepage of the statistics portal, then we just show the summary report
// Otherwise we will show a form to let user enter more information for deeper detail.
if(requestURIElements[requestURIElements.length-1].trim().equalsIgnoreCase(elasticStatisticsPath)) {
//Homepage will show the last 5 years worth of Data, and no form generator.
Calendar cal = Calendar.getInstance();
dateEnd = cal.getTime();
//Roll back to Jan 1 0:00.000 five years ago.
cal.roll(Calendar.YEAR, -5);
cal.set(Calendar.MONTH, 0);
cal.set(Calendar.DAY_OF_MONTH, 1);
cal.set(Calendar.HOUR_OF_DAY,0);
cal.set(Calendar.MINUTE, 0);
cal.set(Calendar.SECOND, 0);
cal.set(Calendar.MILLISECOND, 0);
dateStart = cal.getTime();
division.addHidden("reportDepth").setValue("summary");
String dateRange = "Last Five Years";
division.addPara("Showing Data ( " + dateRange + " )");
division.addHidden("timeRangeString").setValue("Data Range: " + dateRange);
if(dateStart != null) {
division.addHidden("dateStart").setValue(dateFormat.format(dateStart));
}
if(dateEnd != null) {
division.addHidden("dateEnd").setValue(dateFormat.format(dateEnd));
}
showAllReports();
} else {
//Other pages will show a form to choose which date range.
ReportGenerator reportGenerator = new ReportGenerator();
reportGenerator.addReportGeneratorForm(division, request);
dateStart = reportGenerator.getDateStart();
dateEnd = reportGenerator.getDateEnd();
String requestedReport = requestURIElements[requestURIElements.length-1];
log.info("Requested report is: "+ requestedReport);
division.addHidden("reportDepth").setValue("detail");
String dateRange = "";
if(dateStart != null && dateEnd != null) {
dateRange = "from: "+dateFormat.format(dateStart) + " to: "+dateFormat.format(dateEnd);
} else if (dateStart != null && dateEnd == null) {
dateRange = "starting from: "+dateFormat.format(dateStart);
} else if(dateStart == null && dateEnd != null) {
dateRange = "ending with: "+dateFormat.format(dateEnd);
} else if(dateStart == null && dateEnd == null) {
dateRange = "All Data Available";
}
division.addPara("Showing Data ( " + dateRange + " )");
division.addHidden("timeRangeString").setValue(dateRange);
if(dateStart != null) {
division.addHidden("dateStart").setValue(dateFormat.format(dateStart));
}
if(dateEnd != null) {
division.addHidden("dateEnd").setValue(dateFormat.format(dateEnd));
}
division.addHidden("reportName").setValue(requestedReport);
if(requestedReport.equalsIgnoreCase("topCountries"))
{
SearchRequestBuilder requestBuilder = facetedQueryBuilder(facetTopCountries, facetTopUSCities);
searchResponseToDRI(requestBuilder);
}
else if(requestedReport.equalsIgnoreCase("fileDownloads"))
{
SearchRequestBuilder requestBuilder = facetedQueryBuilder(facetMonthlyDownloads);
searchResponseToDRI(requestBuilder);
}
else if(requestedReport.equalsIgnoreCase("topDownloads"))
{
SearchRequestBuilder requestBuilder = facetedQueryBuilder(facetTopBitstreamsAllTime, facetTopBitstreamsLastMonth());
SearchResponse resp = searchResponseToDRI(requestBuilder);
TermsFacet bitstreamsAllTimeFacet = resp.getFacets().facet(TermsFacet.class, "top_bitstreams_alltime");
addTermFacetToTable(bitstreamsAllTimeFacet, division, "Bitstream", "Top Downloads (all time)");
TermsFacet bitstreamsFacet = resp.getFacets().facet(TermsFacet.class, "top_bitstreams_lastmonth");
addTermFacetToTable(bitstreamsFacet, division, "Bitstream", "Top Downloads for " + getLastMonthString());
}
}
} finally {
//client.close();
}
}
public void showAllReports() throws WingException, SQLException{
List<AbstractFacetBuilder> summaryFacets = new ArrayList<AbstractFacetBuilder>();
summaryFacets.add(facetTopTypes);
summaryFacets.add(facetTopUniqueIP);
summaryFacets.add(facetTopCountries);
summaryFacets.add(facetTopUSCities);
summaryFacets.add(facetTopBitstreamsLastMonth());
summaryFacets.add(facetTopBitstreamsAllTime);
summaryFacets.add(facetMonthlyDownloads);
SearchRequestBuilder requestBuilder = facetedQueryBuilder(summaryFacets);
SearchResponse resp = searchResponseToDRI(requestBuilder);
// Top Downloads to Owning Object
TermsFacet bitstreamsFacet = resp.getFacets().facet(TermsFacet.class, "top_bitstreams_lastmonth");
addTermFacetToTable(bitstreamsFacet, division, "Bitstream", "Top Downloads for " + getLastMonthString());
// Convert Elastic Search data to a common DataTermsFacet object, and stuff in DRI/HTML of page.
TermsFacet topBitstreamsFacet = resp.getFacets().facet(TermsFacet.class, "top_bitstreams_lastmonth");
List<? extends TermsFacet.Entry> termsFacetEntries = topBitstreamsFacet.getEntries();
DataTermsFacet termsFacet = new DataTermsFacet();
for(TermsFacet.Entry entry : termsFacetEntries) {
termsFacet.addTermFacet(new DataTermsFacet.TermsFacet(entry.getTerm(), entry.getCount()));
}
division.addHidden("jsonTopDownloads").setValue(termsFacet.toJson());
}
public AbstractFacetBuilder facetTopBitstreamsLastMonth() {
Calendar calendar = Calendar.getInstance();
// Show Previous Whole Month
calendar.add(Calendar.MONTH, -1);
calendar.set(Calendar.DAY_OF_MONTH, calendar.getActualMinimum(Calendar.DAY_OF_MONTH));
String lowerBound = dateFormat.format(calendar.getTime());
calendar.set(Calendar.DAY_OF_MONTH, calendar.getActualMaximum(Calendar.DAY_OF_MONTH));
String upperBound = dateFormat.format(calendar.getTime());
log.info("Lower:"+lowerBound+" -- Upper:"+upperBound);
return FacetBuilders.termsFacet("top_bitstreams_lastmonth").field("id")
.facetFilter(FilterBuilders.andFilter(
FilterBuilders.termFilter("type", "BITSTREAM"),
justOriginals,
FilterBuilders.rangeFilter("time").from(lowerBound).to(upperBound)
));
}
public String getLastMonthString() {
Calendar calendar = Calendar.getInstance();
// Show Previous Whole Month
calendar.add(Calendar.MONTH, -1);
calendar.set(Calendar.DAY_OF_MONTH, calendar.getActualMinimum(Calendar.DAY_OF_MONTH));
return monthAndYearFormat.format(calendar.getTime());
}
public SearchRequestBuilder facetedQueryBuilder(AbstractFacetBuilder facet) throws WingException{
List<AbstractFacetBuilder> facetList = new ArrayList<AbstractFacetBuilder>();
facetList.add(facet);
return facetedQueryBuilder(facetList);
}
public SearchRequestBuilder facetedQueryBuilder(AbstractFacetBuilder... facets) throws WingException {
List<AbstractFacetBuilder> facetList = new ArrayList<AbstractFacetBuilder>();
for(AbstractFacetBuilder facet : facets) {
facetList.add(facet);
}
return facetedQueryBuilder(facetList);
}
public SearchRequestBuilder facetedQueryBuilder(List<AbstractFacetBuilder> facetList) {
TermQueryBuilder termQuery = QueryBuilders.termQuery(getOwningText(dso), dso.getID());
FilterBuilder rangeFilter = FilterBuilders.rangeFilter("time").from(dateStart).to(dateEnd);
FilteredQueryBuilder filteredQueryBuilder = QueryBuilders.filteredQuery(termQuery, rangeFilter);
SearchRequestBuilder searchRequestBuilder = client.prepareSearch(ElasticSearchLogger.getInstance().indexName)
.setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
.setQuery(filteredQueryBuilder)
.setSize(0);
for(AbstractFacetBuilder facet : facetList) {
searchRequestBuilder.addFacet(facet);
}
return searchRequestBuilder;
}
public SearchResponse searchResponseToDRI(SearchRequestBuilder searchRequestBuilder) throws WingException{
division.addHidden("request").setValue(searchRequestBuilder.toString());
SearchResponse resp = searchRequestBuilder.execute().actionGet();
if(resp == null) {
log.info("Elastic Search is down for searching.");
division.addPara("Elastic Search seems to be down :(");
return null;
}
division.addHidden("response").setValue(resp.toString());
division.addDivision("chart_div");
return resp;
}
private void addTermFacetToTable(TermsFacet termsFacet, Division division, String termName, String tableHeader) throws WingException, SQLException {
List<? extends TermsFacet.Entry> termsFacetEntries = termsFacet.getEntries();
if(termName.equalsIgnoreCase("country")) {
division.addDivision("chart_div_map");
}
Table facetTable = division.addTable("facet-"+termName, termsFacetEntries.size()+1, 10);
facetTable.setHead(tableHeader);
Row facetTableHeaderRow = facetTable.addRow(Row.ROLE_HEADER);
if(termName.equalsIgnoreCase("bitstream")) {
facetTableHeaderRow.addCellContent("Title");
facetTableHeaderRow.addCellContent("Creator");
facetTableHeaderRow.addCellContent("Publisher");
facetTableHeaderRow.addCellContent("Date");
} else {
facetTableHeaderRow.addCell().addContent(termName);
}
facetTableHeaderRow.addCell().addContent("Count");
if(termsFacetEntries.size() == 0) {
facetTable.addRow().addCell().addContent("No Data Available");
return;
}
for(TermsFacet.Entry facetEntry : termsFacetEntries) {
Row row = facetTable.addRow();
if(termName.equalsIgnoreCase("bitstream")) {
Bitstream bitstream = Bitstream.find(context, Integer.parseInt(facetEntry.getTerm()));
Item item = (Item) bitstream.getParentObject();
row.addCell().addXref(contextPath + "/handle/" + item.getHandle(), item.getName());
row.addCellContent(getFirstMetadataValue(item, "dc.creator"));
row.addCellContent(getFirstMetadataValue(item, "dc.publisher"));
row.addCellContent(getFirstMetadataValue(item, "dc.date.issued"));
} else if(termName.equalsIgnoreCase("country")) {
row.addCell("country", Cell.ROLE_DATA,"country").addContent(new Locale("en", facetEntry.getTerm()).getDisplayCountry());
} else {
row.addCell().addContent(facetEntry.getTerm());
}
row.addCell("count", Cell.ROLE_DATA, "count").addContent(facetEntry.getCount());
}
}
private void addDateHistogramToTable(DateHistogramFacet monthlyDownloadsFacet, Division division, String termName, String termDescription) throws WingException {
List<? extends DateHistogramFacet.Entry> monthlyFacetEntries = monthlyDownloadsFacet.getEntries();
if(monthlyFacetEntries.size() == 0) {
division.addPara("Empty result set for: "+termName);
return;
}
Table monthlyTable = division.addTable(termName, monthlyFacetEntries.size(), 10);
monthlyTable.setHead(termDescription);
Row tableHeaderRow = monthlyTable.addRow(Row.ROLE_HEADER);
tableHeaderRow.addCell("date", Cell.ROLE_HEADER,null).addContent("Month/Date");
tableHeaderRow.addCell("count", Cell.ROLE_HEADER,null).addContent("Count");
for(DateHistogramFacet.Entry histogramEntry : monthlyFacetEntries) {
Row dataRow = monthlyTable.addRow();
Date facetDate = new Date(histogramEntry.getTime());
dataRow.addCell("date", Cell.ROLE_DATA,"date").addContent(dateFormat.format(facetDate));
dataRow.addCell("count", Cell.ROLE_DATA,"count").addContent("" + histogramEntry.getCount());
}
}
private String getOwningText(DSpaceObject dso) {
switch (dso.getType()) {
case Constants.ITEM:
return "owningItem";
case Constants.COLLECTION:
return "owningColl";
case Constants.COMMUNITY:
return "owningComm";
default:
return "";
}
}
private String getFirstMetadataValue(Item item, String metadataKey) {
DCValue[] dcValue = item.getMetadata(metadataKey);
if(dcValue.length > 0) {
return dcValue[0].value;
} else {
return "";
}
}
}

73
dspace-xmlui/dspace-xmlui-api/src/main/java/org/dspace/app/xmlui/aspect/statisticsElasticSearch/Navigation.java Normal file
View File

@@ -0,0 +1,73 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.xmlui.aspect.statisticsElasticSearch;
import org.dspace.app.xmlui.cocoon.AbstractDSpaceTransformer;
import org.dspace.app.xmlui.wing.element.Options;
import org.dspace.app.xmlui.wing.element.List;
import org.dspace.app.xmlui.wing.WingException;
import org.dspace.app.xmlui.wing.Message;
import org.dspace.app.xmlui.utils.UIException;
import org.dspace.app.xmlui.utils.HandleUtil;
import org.dspace.authorize.AuthorizeException;
import org.dspace.content.DSpaceObject;
import org.apache.cocoon.caching.CacheableProcessingComponent;
import org.apache.excalibur.source.SourceValidity;
import org.apache.excalibur.source.impl.validity.NOPValidity;
import org.xml.sax.SAXException;
import java.io.Serializable;
import java.io.IOException;
import java.sql.SQLException;
/**
* Navigation Elements for viewing Elastic Statistics for Items.
*
*/
public class Navigation extends AbstractDSpaceTransformer implements CacheableProcessingComponent {
private static final Message T_statistics_head = message("xmlui.statistics.Navigation.title");
private static final Message T_statistics_view = message("xmlui.statistics.Navigation.view");
public Serializable getKey() {
//TODO: DO THIS
return null;
}
/**
* Generate the cache validity object.
*
* The cache is always valid.
*/
public SourceValidity getValidity() {
return NOPValidity.SHARED_INSTANCE;
}
/**
* Add the statistics aspect navigational options.
*/
public void addOptions(Options options) throws SAXException, WingException, UIException, SQLException, IOException, AuthorizeException
{
/* Create skeleton menu structure to ensure consistent order between aspects,
* even if they are never used
*/
options.addList("browse");
options.addList("account");
options.addList("context");
options.addList("administrative");
List statistics = options.addList("statistics");
DSpaceObject dso = HandleUtil.obtainHandle(objectModel);
if(dso != null && dso.getHandle() != null){
statistics.setHead(T_statistics_head);
statistics.addItemXref(contextPath + "/handle/" + dso.getHandle() + "/stats", T_statistics_view);
}
}
}

157
dspace-xmlui/dspace-xmlui-api/src/main/java/org/dspace/app/xmlui/aspect/statisticsElasticSearch/ReportGenerator.java Normal file
View File

@@ -0,0 +1,157 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.xmlui.aspect.statisticsElasticSearch;
import org.apache.cocoon.environment.Request;
import org.apache.commons.validator.routines.DateValidator;
import org.apache.log4j.Logger;
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
import org.dspace.app.xmlui.wing.WingException;
import org.dspace.app.xmlui.wing.element.*;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
/**
* Use a form to dynamically generate a variety of reports.
*
* @author "Ryan McGowan" ("mcgowan.98@osu.edu")
* @author Peter Dietz (pdietz84@gmail.com)
* @version
*/
public class ReportGenerator
{
/**
* A logger for this class.
*/
private static Logger log = Logger.getLogger(ReportGenerator.class);
/**
* The minimum date for the from or to field to be. (e.g. The beginning of DSpace)
*/
private static String MINIMUM_DATE = "2008-01-01";
private static SimpleDateFormat dateFormat = new SimpleDateFormat("MM/dd/yyyy");
// perfect input is 2008-01-22, an alternate format is 01/22/2008
static String[] formatStrings = {"MM/dd/yyyy", "yyyy-MM-dd"};
private Map<String, String> params;
private Date dateStart;
private Date dateEnd;
public Date getDateStart() {
return dateStart;
}
public String getDateStartFormated() {
try {
return dateFormat.format(dateStart);
} catch (Exception e) {
return "";
}
}
public void setDateStart() {
if(! params.containsKey("from")) {
dateStart = null;
} else {
dateStart = tryParse(params.get("from"));
}
}
public Date tryParse(String dateString) {
if(dateString == null || dateString.length() == 0) {
return null;
}
for(String formatString : formatStrings) {
try {
return new SimpleDateFormat(formatString).parse(dateString);
} catch (ParseException e) {
log.error("ReportGenerator couldn't parse date: " + dateString + ", with pattern of: "+formatString+" with error message:"+e.getMessage());
}
}
return null;
}
public Date getDateEnd() {
return dateEnd;
}
public String getDateEndFormatted() {
try {
return dateFormat.format(dateEnd);
} catch (Exception e) {
return "";
}
}
public void setDateEnd() {
if(! params.containsKey("to")) {
dateEnd= null;
} else {
dateEnd = tryParse(params.get("to"));
}
}
/**
* {@inheritDoc}
* @see org.dspace.app.xmlui.cocoon.DSpaceTransformer#addBody(Body)
*/
public void addReportGeneratorForm(Division parentDivision, Request request) {
try {
Division division = parentDivision.addDivision("report-generator", "primary");
division.setHead("Report Generator");
division.addPara("Used to generate reports with an arbitrary date range.");
Division search = parentDivision.addInteractiveDivision("choose-report", request.getRequestURI(), Division.METHOD_GET, "primary");
params = new HashMap<String, String>();
for (Enumeration<String> paramNames = (Enumeration<String>) request.getParameterNames(); paramNames.hasMoreElements(); ) {
String param = paramNames.nextElement();
params.put(param, request.getParameter(param));
}
//params = checkAndNormalizeParameters(params);
//Create Date Range part of form
Para reportForm = search.addPara();
setDateStart();
Text from = reportForm.addText("from", "slick");
from.setLabel("From");
from.setHelp("The start date of the report, ex 01/31/2008");
from.setValue(getDateStartFormated());
setDateEnd();
Text to = reportForm.addText("to", "slick");
to.setLabel("To");
to.setHelp("The end date of the report, ex 12/31/2012");
to.setValue(getDateEndFormatted());
//Add whether it is fiscal or not
//CheckBox isFiscal = reportForm.addCheckBox("fiscal", "slick");
//isFiscal.setLabel("Use Fiscal Years?");
//Set up fiscal option with the correct default
//isFiscal.addOption(params.containsKey("fiscal") && params.get("fiscal").equals("1"), 1, "");
reportForm.addButton("submit_add").setValue("Generate Report");
} catch (WingException e) {
log.error(e.getMessage());
}
}
}

57
dspace-xmlui/dspace-xmlui-api/src/main/java/org/dspace/app/xmlui/aspect/statisticsElasticSearch/SpecifiedGroupAuthenticatedSelector.java Normal file
View File

@@ -0,0 +1,57 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.xmlui.aspect.statisticsElasticSearch;
import org.apache.avalon.framework.parameters.Parameters;
import org.apache.cocoon.selection.Selector;
import org.apache.log4j.Logger;
import org.dspace.app.xmlui.utils.ContextUtil;
import org.dspace.core.Context;
import org.dspace.eperson.Group;
import java.sql.SQLException;
import java.util.Map;
/**
* Part of cocoon authentication for resource. Requires that user (eperson) is a member of a specified group.
*
* @author Peter Dietz (pdietz84@gmailcom)
*/
public class SpecifiedGroupAuthenticatedSelector implements Selector {
private static Logger log = Logger.getLogger(SpecifiedGroupAuthenticatedSelector.class);
private String SPECIFIED_GROUP = "statistics_viewer";
@Override
public boolean select(String groupName, Map objectModel, Parameters parameters) {
boolean authorized = false;
if(groupName.equals("statistics_viewer")) {
try
{
Context context = ContextUtil.obtainContext(objectModel);
Group statsGroup = Group.findByName(context, groupName);
if(statsGroup != null && context.getCurrentUser() != null) {
//The Stats Group exists, now lets check that the current user is a member.
if(statsGroup.isMember(context.getCurrentUser())) {
//YES, this person is a member of this group. Let them through.
authorized = true;
}
}
} catch (SQLException e) {
log.error("SQL Error during stats group lookup.");
}
} else {
log.warn("Pattern/test must be statistics_viewer");
}
return authorized;
}
}

144
dspace-xmlui/dspace-xmlui-api/src/main/resources/aspects/StatisticsElasticSearch/sitemap.xmap Normal file
View File

@@ -0,0 +1,144 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
The contents of this file are subject to the license and copyright
detailed in the LICENSE and NOTICE files at the root of the source
tree and available online at
http://www.dspace.org/license/
-->
<!--
The Dashboard Aspect is responsible providing a dashboard for statistical inquiries to the system.
-->
<map:sitemap xmlns:map="http://apache.org/cocoon/sitemap/1.0">
<map:components>
<map:transformers>
<map:transformer name="ElasticSearchStatsViewer" src="org.dspace.app.xmlui.aspect.statisticsElasticSearch.ElasticSearchStatsViewer"/>
<map:transformer name="RestrictedItem" src="org.dspace.app.xmlui.aspect.artifactbrowser.RestrictedItem"/>
<map:transformer name="Navigation" src="org.dspace.app.xmlui.aspect.statisticsElasticSearch.Navigation"/>
</map:transformers>
<map:matchers default="wildcard">
<map:matcher name="HandleTypeMatcher" src="org.dspace.app.xmlui.aspect.general.HandleTypeMatcher"/>
<map:matcher name="StatisticsAuthorizedMatcher" src="org.dspace.app.xmlui.aspect.statistics.StatisticsAuthorizedMatcher"/>
</map:matchers>
<map:selectors>
<map:selector name="AuthenticatedSelector" src="org.dspace.app.xmlui.aspect.general.AuthenticatedSelector"/>
<map:selector name="SpecifiedGroupAuthenticatedSelector" src="org.dspace.app.xmlui.aspect.statisticsElasticSearch.SpecifiedGroupAuthenticatedSelector"/>
</map:selectors>
</map:components>
<map:pipelines>
<map:pipeline>
<map:generate/>
<!--Only show the stats link if we have read rights-->
<map:match pattern="handle/*/**">
<map:match type="StatisticsAuthorizedMatcher" pattern="READ">
<map:transform type="Navigation"/>
</map:match>
</map:match>
<map:match pattern="handle/*/*/stats">
<map:select type="SpecifiedGroupAuthenticatedSelector">
<map:when test="statistics_viewer">
<map:transform type="IncludePageMeta">
<map:parameter name="stylesheet.screen.statisticsElasticSearch#1" value="../../static/css/statisticsElasticSearch/style.css"/>
<map:parameter name="javascript.url#1" value="https://www.google.com/jsapi" />
<map:parameter name="javascript.static#1" value="loadJQuery.js"/>
<map:parameter name="javascript.static#2" value="static/js/statisticsElasticSearch/jquery.ui.datepicker-accessible.min.js"/>
<map:parameter name="javascript.static#3" value="static/js/statisticsElasticSearch/visualizeData.js"/>
</map:transform>
<map:transform type="ElasticSearchStatsViewer"/>
</map:when>
<map:otherwise>
<map:match type="StatisticsAuthorizedMatcher" pattern="READ">
<map:transform type="IncludePageMeta">
<map:parameter name="stylesheet.screen.statisticsElasticSearch#1" value="../../static/css/statisticsElasticSearch/style.css"/>
<map:parameter name="javascript.url#1" value="https://www.google.com/jsapi" />
<map:parameter name="javascript.static#1" value="loadJQuery.js"/>
<map:parameter name="javascript.static#2" value="static/js/statisticsElasticSearch/jquery.ui.datepicker-accessible.min.js"/>
<map:parameter name="javascript.static#3" value="static/js/statisticsElasticSearch/visualizeData.js"/>
</map:transform>
<map:transform type="ElasticSearchStatsViewer"/>
</map:match>
<map:match type="StatisticsAuthorizedMatcher" pattern="!READ">
<map:select type="SpecifiedGroupAuthenticatedSelector">
<map:when test="statistics_viewer">
<map:transform type="RestrictedItem"/>
<map:serialize/>
</map:when>
<map:otherwise>
<map:act type="StartAuthentication">
<map:parameter name="header" value="xmlui.ArtifactBrowser.RestrictedItem.auth_header"/>
<map:parameter name="message" value="xmlui.ArtifactBrowser.RestrictedItem.auth_message"/>
</map:act>
<map:serialize/>
</map:otherwise>
</map:select>
</map:match>
<map:serialize type="xml"/>
</map:otherwise>
</map:select>
</map:match>
<map:match pattern="handle/*/*/stats/*">
<map:select type="SpecifiedGroupAuthenticatedSelector">
<map:when test="statistics_viewer">
<map:transform type="IncludePageMeta">
<map:parameter name="stylesheet.screen.statisticsElasticSearch#1" value="../../static/css/statisticsElasticSearch/style.css"/>
<map:parameter name="javascript.url#1" value="https://www.google.com/jsapi" />
<map:parameter name="javascript.static#1" value="loadJQuery.js"/>
<map:parameter name="javascript.static#2" value="static/js/statisticsElasticSearch/jquery.ui.datepicker-accessible.min.js"/>
<map:parameter name="javascript.static#3" value="static/js/statisticsElasticSearch/visualizeData.js"/>
</map:transform>
<map:transform type="ElasticSearchStatsViewer"/>
</map:when>
<map:otherwise>
<map:match type="StatisticsAuthorizedMatcher" pattern="READ">
<map:transform type="IncludePageMeta">
<map:parameter name="stylesheet.screen.statisticsElasticSearch#1" value="../../static/css/statisticsElasticSearch/style.css"/>
<map:parameter name="javascript.url#1" value="https://www.google.com/jsapi" />
<map:parameter name="javascript.static#1" value="loadJQuery.js"/>
<map:parameter name="javascript.static#2" value="static/js/statisticsElasticSearch/jquery.ui.datepicker-accessible.min.js"/>
<map:parameter name="javascript.static#3" value="static/js/statisticsElasticSearch/visualizeData.js"/>
</map:transform>
<map:transform type="ElasticSearchStatsViewer"/>
</map:match>
<map:match type="StatisticsAuthorizedMatcher" pattern="!READ">
<map:select type="AuthenticatedSelector">
<map:when test="eperson">
<map:transform type="RestrictedItem"/>
<map:serialize/>
</map:when>
<map:otherwise>
<map:act type="StartAuthentication">
<map:parameter name="header" value="xmlui.ArtifactBrowser.RestrictedItem.auth_header"/>
<map:parameter name="message" value="xmlui.ArtifactBrowser.RestrictedItem.auth_message"/>
</map:act>
<map:serialize/>
</map:otherwise>
</map:select>
</map:match>
<map:serialize type="xml"/>
</map:otherwise>
</map:select>
</map:match>
<!-- Not a URL we care about, so just pass it on. -->
<map:serialize type="xml"/>
</map:pipeline>
</map:pipelines>
</map:sitemap>

7
dspace-xmlui/dspace-xmlui-webapp/src/main/webapp/WEB-INF/spring/applicationContext.xml
View File

@@ -50,6 +50,13 @@
<ref bean="dspace.eventService"/>
</property>
</bean>
<!-- Elastic Search -->
<!--<bean class="org.dspace.statistics.ElasticSearchLoggerEventListener">
<property name="eventService">
<ref bean="dspace.eventService" />
</property>
</bean>-->
<!--
Uncomment to enable TabFileUsageEventListener

51
dspace-xmlui/dspace-xmlui-webapp/src/main/webapp/sitemap.xmap
View File

@@ -95,6 +95,7 @@
<map:transformer name="LDAPLogin" src="org.dspace.app.xmlui.aspect.eperson.LDAPLogin"/>
<map:transformer name="notice" src="org.dspace.app.xmlui.aspect.general.NoticeTransformer"/>
<map:transformer name="NamespaceFilter" src="org.dspace.app.xmlui.cocoon.NamespaceFilterTransformer"/>
<map:transformer name="RestrictedItem" src="org.dspace.app.xmlui.aspect.artifactbrowser.RestrictedItem"/>
</map:transformers>
<map:matchers default="wildcard">
<map:matcher logger="sitemap.matcher.wildcard" name="wildcard"
@@ -122,6 +123,7 @@
<map:parameter name="ignore-missing-tables" value="true"/>
</map:matcher>
<map:matcher name="request" src="org.apache.cocoon.matching.RequestParameterMatcher"/>
<map:matcher name="StatisticsAuthorizedMatcher" src="org.dspace.app.xmlui.aspect.statistics.StatisticsAuthorizedMatcher"/>
</map:matchers>
<map:selectors>
<map:selector name="browser" src="org.apache.cocoon.selection.BrowserSelector"
@@ -177,6 +179,8 @@
<!-- The statement below tells the selector to unroll as much exceptions as possible -->
<exception class="java.lang.Throwable" unroll="true"/>
</map:selector>
<!-- AuthenticatedSelector for forcing user to login if not authorized - PMD -->
<map:selector name="AuthenticatedSelector" src="org.dspace.app.xmlui.aspect.general.AuthenticatedSelector"/>
</map:selectors>
<map:readers default="resource">
<map:reader name="resource" src="org.apache.cocoon.reading.ResourceReader"
@@ -186,6 +190,9 @@
<map:reader name="BitstreamReader" src="org.dspace.app.xmlui.cocoon.BitstreamReader"/>
<map:reader name="ExportReader" src="org.dspace.app.xmlui.cocoon.ItemExportDownloadReader"/>
<map:reader name="MetadataExportReader" src="org.dspace.app.xmlui.cocoon.MetadataExportReader"/>
<map:reader name="CSVOutputter" src="org.dspace.app.xmlui.aspect.statisticsElasticSearch.CSVOutputter"/>
<map:reader name="OpenURLReader" src="org.dspace.app.xmlui.cocoon.OpenURLReader"/>
<map:reader name="SitemapReader" src="org.dspace.app.xmlui.cocoon.SitemapReader"/>
<map:reader name="ConcatenationReader" src="org.dspace.app.xmlui.cocoon.ConcatenationReader"/>
@@ -378,6 +385,50 @@
</map:read>
</map:match>
<map:match pattern="handle/*/*/stats/csv">
<map:match type="StatisticsAuthorizedMatcher" pattern="READ">
<map:read type="CSVOutputter"/>
</map:match>
<map:match type="StatisticsAuthorizedMatcher" pattern="!READ">
<map:select type="AuthenticatedSelector">
<map:when test="eperson">
<map:transform type="RestrictedItem"/>
<map:serialize/>
</map:when>
<map:otherwise>
<map:act type="StartAuthentication">
<map:parameter name="header" value="xmlui.ArtifactBrowser.RestrictedItem.auth_header"/>
<map:parameter name="message" value="xmlui.ArtifactBrowser.RestrictedItem.auth_message"/>
</map:act>
<map:serialize/>
</map:otherwise>
</map:select>
</map:match>
</map:match>
<map:match pattern="handle/*/*/stats/csv/*">
<map:match type="StatisticsAuthorizedMatcher" pattern="READ">
<map:read type="CSVOutputter"/>
</map:match>
<map:match type="StatisticsAuthorizedMatcher" pattern="!READ">
<map:select type="AuthenticatedSelector">
<map:when test="eperson">
<map:transform type="RestrictedItem"/>
<map:serialize/>
</map:when>
<map:otherwise>
<map:act type="StartAuthentication">
<map:parameter name="header" value="xmlui.ArtifactBrowser.RestrictedItem.auth_header"/>
<map:parameter name="message" value="xmlui.ArtifactBrowser.RestrictedItem.auth_message"/>
</map:act>
<map:serialize/>
</map:otherwise>
</map:select>
</map:match>
</map:match>
<map:match pattern="JSON/discovery/**">
<map:mount check-reload="no" src="aspects/json-solr.xmap" uri-prefix="JSON/discovery/"/>
</map:match>

24
dspace-xmlui/dspace-xmlui-webapp/src/main/webapp/static/css/statisticsElasticSearch/style.css Normal file
View File

@@ -0,0 +1,24 @@
#chart_div {
/*width: 700px;*/
/*height: 240px;*/
}
#aspect_artifactbrowser_DashboardViewer_table_items_added_monthly {
display: none;
}
.small-icon:before {
padding-right: 5px;
}
.left-arrow-icon:before {
content: url(../../../static/icons/arrow_left.png);
}
.print-icon:before {
content: url(../../../static/icons/printer.png);
}
.excel-icon:before {
content: url(../../../static/icons/page_excel.png);
}

460
dspace-xmlui/dspace-xmlui-webapp/src/main/webapp/static/js/statisticsElasticSearch/gviz-api.js Normal file
View File

@@ -0,0 +1,460 @@
// Copyright 2009 Google Inc.
// All Rights Reserved.
/**
* This file exposes the external Google Visualization API.
*
* The file can be used to enable auto complete of objects and methods provided by the
* Google Visualization API, and for easier exploration of the API.
*
* To enable auto complete in a development environment - copy the file into the project
* you are working on where the development tool you are using can index the file.
*
* Disclaimer: there may be missing classes and methods and the file may
* be updated and/or changed. For the most up to date API reference please visit:
* {@link http://code.google.com/intl/iw/apis/visualization/documentation/reference.html}
*/
var google = {};
google.visualization = {};
/** @constructor */
google.visualization.DataTable = function(opt_data, opt_version) {};
google.visualization.DataTable.prototype.getNumberOfRows = function() {};
google.visualization.DataTable.prototype.getNumberOfColumns = function() {};
google.visualization.DataTable.prototype.clone = function() {};
google.visualization.DataTable.prototype.getColumnId = function(columnIndex) {};
google.visualization.DataTable.prototype.getColumnIndex = function(columnId) {};
google.visualization.DataTable.prototype.getColumnLabel = function(columnIndex) {};
google.visualization.DataTable.prototype.getColumnPattern = function(columnIndex) {};
google.visualization.DataTable.prototype.getColumnRole = function(columnIndex) {};
google.visualization.DataTable.prototype.getColumnType = function(columnIndex) {};
google.visualization.DataTable.prototype.getValue = function(rowIndex, columnIndex) {};
google.visualization.DataTable.prototype.getFormattedValue = function(rowIndex, columnIndex) {};
google.visualization.DataTable.prototype.getProperty = function(rowIndex, columnIndex, property) {};
google.visualization.DataTable.prototype.getProperties = function(rowIndex, columnIndex) {};
google.visualization.DataTable.prototype.getTableProperties = function() {};
google.visualization.DataTable.prototype.getTableProperty = function(property) {};
google.visualization.DataTable.prototype.setTableProperties = function(properties) {};
google.visualization.DataTable.prototype.setTableProperty = function(property, value) {};
google.visualization.DataTable.prototype.setValue = function(rowIndex, columnIndex, value) {};
google.visualization.DataTable.prototype.setFormattedValue = function(rowIndex, columnIndex, formattedValue) {};
google.visualization.DataTable.prototype.setProperties = function(rowIndex, columnIndex, properties) {};
google.visualization.DataTable.prototype.setProperty = function(rowIndex, columnIndex, property, value) {};
google.visualization.DataTable.prototype.setCell = function(rowIndex, columnIndex, opt_value, opt_formattedValue, opt_properties) {};
google.visualization.DataTable.prototype.setRowProperties = function(rowIndex, properties) {};
google.visualization.DataTable.prototype.setRowProperty = function(rowIndex, property, value) {};
google.visualization.DataTable.prototype.getRowProperty = function(rowIndex, property) {};
google.visualization.DataTable.prototype.getRowProperties = function(rowIndex) {};
google.visualization.DataTable.prototype.setColumnLabel = function(columnIndex, newLabel) {};
google.visualization.DataTable.prototype.setColumnProperties = function(columnIndex, properties) {};
google.visualization.DataTable.prototype.setColumnProperty = function(columnIndex, property, value) {};
google.visualization.DataTable.prototype.getColumnProperty = function(columnIndex, property) {};
google.visualization.DataTable.prototype.getColumnProperties = function(columnIndex) {};
google.visualization.DataTable.prototype.insertColumn = function(atColIndex, type, opt_label, opt_id) {};
google.visualization.DataTable.prototype.addColumn = function(type, opt_label, opt_id) {};
google.visualization.DataTable.prototype.insertRows = function(atRowIndex, numOrArray) {};
google.visualization.DataTable.prototype.addRows = function(numOrArray) {};
google.visualization.DataTable.prototype.addRow = function(opt_cellArray) {};
google.visualization.DataTable.prototype.getColumnRange = function(columnIndex) {};
google.visualization.DataTable.prototype.getSortedRows = function(sortColumns) {};
google.visualization.DataTable.prototype.sort = function(sortColumns) {};
google.visualization.DataTable.prototype.getDistinctValues = function(column) {};
google.visualization.DataTable.prototype.getFilteredRows = function(columnFilters) {};
google.visualization.DataTable.prototype.removeRows = function(fromRowIndex, numRows) {};
google.visualization.DataTable.prototype.removeRow = function(rowIndex) {};
google.visualization.DataTable.prototype.removeColumns = function(fromColIndex, numCols) {};
google.visualization.DataTable.prototype.removeColumn = function(colIndex) {};
/** @return {string} JSON representation. */
google.visualization.DataTable.prototype.toJSON = function() {};
/** @constructor */
google.visualization.QueryResponse = function(responseObj) {};
google.visualization.QueryResponse.getVersionFromResponse = function(responseObj) {};
google.visualization.QueryResponse.prototype.getVersion = function() {};
google.visualization.QueryResponse.prototype.getExecutionStatus = function() {};
google.visualization.QueryResponse.prototype.isError = function() {};
google.visualization.QueryResponse.prototype.hasWarning = function() {};
google.visualization.QueryResponse.prototype.containsReason = function(reason) {};
google.visualization.QueryResponse.prototype.getDataSignature = function() {};
google.visualization.QueryResponse.prototype.getDataTable = function() {};
google.visualization.QueryResponse.prototype.getReasons = function() {};
google.visualization.QueryResponse.prototype.getMessage = function() {};
google.visualization.QueryResponse.prototype.getDetailedMessage = function() {};
/** @constructor */
google.visualization.Query = function(dataSourceUrl, opt_options) {};
google.visualization.Query.refreshAllQueries = function() {};
google.visualization.Query.setResponse = function(response) {};
google.visualization.Query.prototype.setRefreshInterval = function(intervalSeconds) {};
google.visualization.Query.prototype.send = function(responseHandler) {};
google.visualization.Query.prototype.makeRequest = function(responseHandler, opt_params) {};
google.visualization.Query.prototype.abort = function() {};
google.visualization.Query.prototype.setTimeout = function(timeoutSeconds) {};
google.visualization.Query.prototype.setRefreshable = function(refreshable) {};
google.visualization.Query.prototype.setQuery = function(queryString) {};
google.visualization.errors = {};
google.visualization.errors.addError = function(container, message, opt_detailedMessage, opt_options) {};
google.visualization.errors.removeAll = function(container) {};
google.visualization.errors.addErrorFromQueryResponse = function(container, response) {};
google.visualization.errors.removeError = function(id) {};
google.visualization.errors.getContainer = function(errorId) {};
google.visualization.events = {};
google.visualization.events.addListener = function(eventSource, eventName, eventHandler) {};
google.visualization.events.trigger = function(eventSource, eventName, eventDetails) {};
google.visualization.events.removeListener = function(listener) {};
google.visualization.events.removeAllListeners = function(eventSource) {};
/** @constructor */
google.visualization.DataView = function(dataTable) {};
google.visualization.DataView.fromJSON = function(dataTable, view) {};
google.visualization.DataView.prototype.setColumns = function(colIndices) {};
google.visualization.DataView.prototype.setRows = function(arg0, opt_arg1) {};
google.visualization.DataView.prototype.getViewColumns = function() {};
google.visualization.DataView.prototype.getViewRows = function() {};
google.visualization.DataView.prototype.hideColumns = function(colIndices) {};
google.visualization.DataView.prototype.hideRows = function(arg0, opt_arg1) {};
google.visualization.DataView.prototype.getViewColumnIndex = function(tableColumnIndex) {};
google.visualization.DataView.prototype.getViewRowIndex = function(tableRowIndex) {};
google.visualization.DataView.prototype.getTableColumnIndex = function(viewColumnIndex) {};
google.visualization.DataView.prototype.getUnderlyingTableColumnIndex = function(viewColumnIndex) {};
google.visualization.DataView.prototype.getTableRowIndex = function(viewRowIndex) {};
google.visualization.DataView.prototype.getUnderlyingTableRowIndex = function(viewRowIndex) {};
google.visualization.DataView.prototype.getNumberOfRows = function() {};
google.visualization.DataView.prototype.getNumberOfColumns = function() {};
google.visualization.DataView.prototype.getColumnId = function(columnIndex) {};
google.visualization.DataView.prototype.getColumnIndex = function(columnId) {};
google.visualization.DataView.prototype.getColumnLabel = function(columnIndex) {};
google.visualization.DataView.prototype.getColumnPattern = function(columnIndex) {};
google.visualization.DataView.prototype.getColumnRole = function(columnIndex) {};
google.visualization.DataView.prototype.getColumnType = function(columnIndex) {};
google.visualization.DataView.prototype.getValue = function(rowIndex, columnIndex) {};
google.visualization.DataView.prototype.getFormattedValue = function(rowIndex, columnIndex) {};
google.visualization.DataView.prototype.getProperty = function(rowIndex, columnIndex, property) {};
google.visualization.DataView.prototype.getColumnProperty = function(columnIndex, property) {};
google.visualization.DataView.prototype.getColumnProperties = function(columnIndex) {};
google.visualization.DataView.prototype.getTableProperty = function(property) {};
google.visualization.DataView.prototype.getTableProperties = function() {};
google.visualization.DataView.prototype.getRowProperty = function(rowIndex, property) {};
google.visualization.DataView.prototype.getRowProperties = function(rowIndex) {};
google.visualization.DataView.prototype.getColumnRange = function(columnIndex) {};
google.visualization.DataView.prototype.getDistinctValues = function(columnIndex) {};
google.visualization.DataView.prototype.getSortedRows = function(sortColumns) {};
google.visualization.DataView.prototype.getFilteredRows = function(columnFilters) {};
google.visualization.DataView.prototype.toDataTable = function() {};
/** @return {string} JSON representation. */
google.visualization.DataView.prototype.toJSON = function() {};
/** @constructor */
google.visualization.ArrowFormat = function(opt_options) {};
google.visualization.ArrowFormat.prototype.format = function(dataTable, columnIndex) {};
/** @constructor */
google.visualization.BarFormat = function(opt_options) {};
google.visualization.BarFormat.prototype.format = function(dataTable, columnIndex) {};
/** @constructor */
google.visualization.ColorFormat = function() {};
google.visualization.ColorFormat.prototype.addRange = function(from, to, color, bgcolor) {};
google.visualization.ColorFormat.prototype.addGradientRange = function(from, to, color, fromBgColor, toBgColor) {};
google.visualization.ColorFormat.prototype.format = function(dataTable, columnIndex) {};
/** @constructor */
google.visualization.DateFormat = function(opt_options) {};
google.visualization.DateFormat.prototype.format = function(dataTable, columnIndex) {};
google.visualization.DateFormat.prototype.formatValue = function(value) {};
/** @constructor */
google.visualization.NumberFormat = function(opt_options) {};
google.visualization.NumberFormat.prototype.format = function(dataTable, columnIndex) {};
google.visualization.NumberFormat.prototype.formatValue = function(value) {};
google.visualization.NumberFormat.DECIMAL_SEP;
google.visualization.NumberFormat.GROUP_SEP;
google.visualization.NumberFormat.DECIMAL_PATTERN;
/** @constructor */
google.visualization.PatternFormat = function(pattern) {};
google.visualization.PatternFormat.prototype.format = function(dataTable, srcColumnIndices, opt_dstColumnIndex) {};
/** @constructor */
google.visualization.GadgetHelper = function() {};
google.visualization.GadgetHelper.prototype.createQueryFromPrefs = function(prefs) {};
google.visualization.GadgetHelper.prototype.validateResponse = function(response) {};
/** @constructor */
google.visualization.AnnotatedTimeLine = function(container) {};
google.visualization.AnnotatedTimeLine.prototype.draw = function(data, opt_options) {};
google.visualization.AnnotatedTimeLine.prototype.getSelection = function() {};
google.visualization.AnnotatedTimeLine.prototype.getVisibleChartRange = function() {};
google.visualization.AnnotatedTimeLine.prototype.setVisibleChartRange = function(firstDate, lastDate, opt_animate) {};
google.visualization.AnnotatedTimeLine.prototype.showDataColumns = function(columnIndexes) {};
google.visualization.AnnotatedTimeLine.prototype.hideDataColumns = function(columnIndexes) {};
/** @constructor */
google.visualization.AreaChart = function(container) {};
google.visualization.AreaChart.prototype.draw = function(data, opt_options, opt_state) {};
google.visualization.AreaChart.prototype.clearChart = function() {};
google.visualization.AreaChart.prototype.getSelection = function() {};
google.visualization.AreaChart.prototype.setSelection = function(selection) {};
/** @constructor */
google.visualization.BarChart = function(container) {};
google.visualization.BarChart.prototype.draw = function(data, opt_options, opt_state) {};
google.visualization.BarChart.prototype.clearChart = function() {};
google.visualization.BarChart.prototype.getSelection = function() {};
google.visualization.BarChart.prototype.setSelection = function(selection) {};
/** @constructor */
google.visualization.BubbleChart = function(container) {};
google.visualization.BubbleChart.prototype.draw = function(data, opt_options, opt_state) {};
google.visualization.BubbleChart.prototype.clearChart = function() {};
google.visualization.BubbleChart.prototype.getSelection = function() {};
google.visualization.BubbleChart.prototype.setSelection = function(selection) {};
/** @constructor */
google.visualization.CandlestickChart = function(container) {};
google.visualization.CandlestickChart.prototype.draw = function(data, opt_options, opt_state) {};
google.visualization.CandlestickChart.prototype.clearChart = function() {};
google.visualization.CandlestickChart.prototype.getSelection = function() {};
google.visualization.CandlestickChart.prototype.setSelection = function(selection) {};
/** @constructor */
google.visualization.ColumnChart = function(container) {};
google.visualization.ColumnChart.prototype.draw = function(data, opt_options, opt_state) {};
google.visualization.ColumnChart.prototype.clearChart = function() {};
google.visualization.ColumnChart.prototype.getSelection = function() {};
google.visualization.ColumnChart.prototype.setSelection = function(selection) {};
/** @constructor */
google.visualization.ComboChart = function(container) {};
google.visualization.ComboChart.prototype.draw = function(data, opt_options, opt_state) {};
google.visualization.ComboChart.prototype.clearChart = function() {};
google.visualization.ComboChart.prototype.getSelection = function() {};
google.visualization.ComboChart.prototype.setSelection = function(selection) {};
/** @constructor */
google.visualization.Gauge = function(container) {};
google.visualization.Gauge.prototype.draw = function(dataTable, opt_options) {};
google.visualization.Gauge.prototype.clearChart = function() {};
/** @constructor */
google.visualization.GeoChart = function(container) {};
google.visualization.GeoChart.mapExists = function(userOptions) {};
google.visualization.GeoChart.prototype.clearChart = function() {};
google.visualization.GeoChart.prototype.draw = function(dataTable, userOptions, opt_state) {};
google.visualization.GeoChart.prototype.getSelection = function() {};
google.visualization.GeoChart.prototype.setSelection = function(selection) {};
/** @constructor */
google.visualization.GeoMap = function(container) {};
google.visualization.GeoMap.clickOnRegion = function(id, zoomLevel, segmentBy, instanceIndex) {};
google.visualization.GeoMap.prototype.draw = function(dataTable, opt_options) {};
google.visualization.GeoMap.prototype.getSelection = function() {};
google.visualization.GeoMap.prototype.setSelection = function(selection) {};
/** @constructor */
google.visualization.Map = function(container) {};
google.visualization.Map.prototype.draw = function(dataTable, opt_options) {};
google.visualization.Map.prototype.getSelection = function() {};
google.visualization.Map.prototype.setSelection = function(selection) {};
/** @constructor */
google.visualization.ImageAreaChart = function(container) {};
google.visualization.ImageAreaChart.prototype.draw = function(data, opt_options) {};
/** @constructor */
google.visualization.ImageBarChart = function(container) {};
google.visualization.ImageBarChart.prototype.draw = function(data, opt_options) {};
/** @constructor */
google.visualization.ImageCandlestickChart = function(container) {};
google.visualization.ImageCandlestickChart.prototype.draw = function(data, opt_options) {};
/** @constructor */
google.visualization.ImageChart = function(container) {};
google.visualization.ImageChart.prototype.draw = function(data, opt_options) {};
/** @constructor */
google.visualization.ImageLineChart = function(container) {};
google.visualization.ImageLineChart.prototype.draw = function(data, opt_options) {};
/** @constructor */
google.visualization.ImagePieChart = function(container) {};
google.visualization.ImagePieChart.prototype.draw = function(data, opt_options) {};
/** @constructor */
google.visualization.ImageSparkLine = function(container, opt_domHelper) {};
google.visualization.ImageSparkLine.prototype.draw = function(dataTable, opt_options) {};
google.visualization.ImageSparkLine.prototype.getSelection = function() {};
google.visualization.ImageSparkLine.prototype.setSelection = function(selection) {};
/** @constructor */
google.visualization.IntensityMap = function(container) {};
google.visualization.IntensityMap.prototype.draw = function(dataTable, opt_options) {};
google.visualization.IntensityMap.prototype.getSelection = function() {};
google.visualization.IntensityMap.prototype.setSelection = function(selection) {};
/** @constructor */
google.visualization.LineChart = function(container) {};
google.visualization.LineChart.prototype.draw = function(data, opt_options, opt_state) {};
google.visualization.LineChart.prototype.clearChart = function() {};
google.visualization.LineChart.prototype.getSelection = function() {};
google.visualization.LineChart.prototype.setSelection = function(selection) {};
/** @constructor */
google.visualization.MotionChart = function(container) {};
google.visualization.MotionChart.prototype.draw = function(dataTable, opt_options) {};
google.visualization.MotionChart.prototype.getState = function() {};
/** @constructor */
google.visualization.OrgChart = function(container) {};
google.visualization.OrgChart.prototype.draw = function(dataTable, opt_options) {};
google.visualization.OrgChart.prototype.getSelection = function() {};
google.visualization.OrgChart.prototype.setSelection = function(selection) {};
google.visualization.OrgChart.prototype.getCollapsedNodes = function() {};
google.visualization.OrgChart.prototype.getChildrenIndexes = function(rowInd) {};
google.visualization.OrgChart.prototype.collapse = function(rowInd, collapse) {};
/** @constructor */
google.visualization.PieChart = function(container) {};
google.visualization.PieChart.prototype.draw = function(data, opt_options, opt_state) {};
google.visualization.PieChart.prototype.clearChart = function() {};
google.visualization.PieChart.prototype.getSelection = function() {};
google.visualization.PieChart.prototype.setSelection = function(selection) {};
/** @constructor */
google.visualization.ScatterChart = function(container) {};
google.visualization.ScatterChart.prototype.draw = function(data, opt_options, opt_state) {};
google.visualization.ScatterChart.prototype.clearChart = function() {};
google.visualization.ScatterChart.prototype.getSelection = function() {};
google.visualization.ScatterChart.prototype.setSelection = function(selection) {};
/** @constructor */
google.visualization.SparklineChart = function(container) {};
google.visualization.SparklineChart.prototype.draw = function(data, opt_options, opt_state) {};
google.visualization.SparklineChart.prototype.clearChart = function() {};
google.visualization.SparklineChart.prototype.getSelection = function() {};
google.visualization.SparklineChart.prototype.setSelection = function(selection) {};
/** @constructor */
google.visualization.SteppedAreaChart = function(container) {};
google.visualization.SteppedAreaChart.prototype.draw = function(data, opt_options, opt_state) {};
google.visualization.SteppedAreaChart.prototype.clearChart = function() {};
google.visualization.SteppedAreaChart.prototype.getSelection = function() {};
google.visualization.SteppedAreaChart.prototype.setSelection = function(selection) {};
/** @constructor */
google.visualization.Table = function(container) {};
google.visualization.Table.prototype.draw = function(dataTable, opt_options) {};
google.visualization.Table.prototype.clearChart = function() {};
google.visualization.Table.prototype.getSortInfo = function() {};
google.visualization.Table.prototype.getSelection = function() {};
google.visualization.Table.prototype.setSelection = function(selection) {};
/** @constructor */
google.visualization.TreeMap = function(container) {};
google.visualization.TreeMap.prototype.draw = function(dataTable, opt_options) {};
google.visualization.TreeMap.prototype.clearChart = function() {};
google.visualization.TreeMap.prototype.getSelection = function() {};
google.visualization.TreeMap.prototype.setSelection = function(selection) {};
google.visualization.drawToolbar = function(container, components) {};
/** @constructor */
google.visualization.ChartWrapper = function(opt_specification) {};
google.visualization.ChartWrapper.prototype.draw = function(opt_container) {};
google.visualization.ChartWrapper.prototype.getDataSourceUrl = function() {};
google.visualization.ChartWrapper.prototype.getDataTable = function() {};
google.visualization.ChartWrapper.prototype.getChartName = function() {};
google.visualization.ChartWrapper.prototype.getChartType = function() {};
google.visualization.ChartWrapper.prototype.getContainerId = function() {};
google.visualization.ChartWrapper.prototype.getQuery = function() {};
google.visualization.ChartWrapper.prototype.getRefreshInterval = function() {};
google.visualization.ChartWrapper.prototype.getView = function() {};
google.visualization.ChartWrapper.prototype.getOption = function(key, opt_default) {};
google.visualization.ChartWrapper.prototype.getOptions = function() {};
google.visualization.ChartWrapper.prototype.setDataSourceUrl = function(dataSourceUrl) {};
google.visualization.ChartWrapper.prototype.setDataTable = function(dataTable) {};
google.visualization.ChartWrapper.prototype.setChartName = function(chartName) {};
google.visualization.ChartWrapper.prototype.setChartType = function(chartType) {};
google.visualization.ChartWrapper.prototype.setContainerId = function(containerId) {};
google.visualization.ChartWrapper.prototype.setQuery = function(query) {};
google.visualization.ChartWrapper.prototype.setRefreshInterval = function(refreshInterval) {};
google.visualization.ChartWrapper.prototype.setView = function(view) {};
google.visualization.ChartWrapper.prototype.setOption = function(key, value) {};
google.visualization.ChartWrapper.prototype.setOptions = function(options) {};
/** @return {string} JSON representation. */
google.visualization.ChartWrapper.prototype.toJSON = function() {};
/** @constructor */
google.visualization.ControlWrapper = function(opt_specification) {};
google.visualization.ControlWrapper.prototype.draw = function(opt_container) {};
google.visualization.ControlWrapper.prototype.toJSON = function() {};
google.visualization.ControlWrapper.prototype.getDataSourceUrl = function() {};
google.visualization.ControlWrapper.prototype.getDataTable = function() {};
google.visualization.ControlWrapper.prototype.getControlName = function() {};
google.visualization.ControlWrapper.prototype.getControlType = function() {};
google.visualization.ControlWrapper.prototype.getContainerId = function() {};
google.visualization.ControlWrapper.prototype.getQuery = function() {};
google.visualization.ControlWrapper.prototype.getRefreshInterval = function() {};
google.visualization.ControlWrapper.prototype.getView = function() {};
google.visualization.ControlWrapper.prototype.getOption = function(key, opt_default) {};
google.visualization.ControlWrapper.prototype.getOptions = function() {};
google.visualization.ControlWrapper.prototype.setDataSourceUrl = function(dataSourceUrl) {};
google.visualization.ControlWrapper.prototype.setDataTable = function(dataTable) {};
google.visualization.ControlWrapper.prototype.setControlName = function(controlName) {};
google.visualization.ControlWrapper.prototype.setControlType = function(controlType) {};
google.visualization.ControlWrapper.prototype.setContainerId = function(containerId) {};
google.visualization.ControlWrapper.prototype.setQuery = function(query) {};
google.visualization.ControlWrapper.prototype.setRefreshInterval = function(refreshInterval) {};
google.visualization.ControlWrapper.prototype.setView = function(view) {};
google.visualization.ControlWrapper.prototype.setOption = function(key, value) {};
google.visualization.ControlWrapper.prototype.setOptions = function(options) {};
/** @return {string} JSON representation. */
google.visualization.ChartWrapper.prototype.toJSON = function() {};
/** @constructor */
google.visualization.ChartEditor = function(opt_config) {};
google.visualization.ChartEditor.prototype.openDialog = function(specification, opt_options) {};
google.visualization.ChartEditor.prototype.getChartWrapper = function() {};
google.visualization.ChartEditor.prototype.setChartWrapper = function(chartWrapper) {};
google.visualization.ChartEditor.prototype.closeDialog = function() {};
/** @constructor */
google.visualization.Dashboard = function(container) {};
google.visualization.Dashboard.prototype.bind = function(controls, participants) {};
google.visualization.Dashboard.prototype.draw = function(dataTable) {};
/** @constructor */
google.visualization.StringFilter = function(container) {};
google.visualization.StringFilter.prototype.draw = function(dataTable, opt_options, opt_state) {};
google.visualization.StringFilter.prototype.applyFilter = function() {};
google.visualization.StringFilter.prototype.getState = function() {};
google.visualization.StringFilter.prototype.resetControl = function() {};
/** @constructor */
google.visualization.NumberRangeFilter = function(container) {};
google.visualization.NumberRangeFilter.prototype.draw = function(dataTable, opt_options, opt_state) {};
google.visualization.NumberRangeFilter.prototype.applyFilter = function() {};
google.visualization.NumberRangeFilter.prototype.getState = function() {};
google.visualization.NumberRangeFilter.prototype.resetControl = function() {};
/** @constructor */
google.visualization.CategoryFilter = function(container) {};
google.visualization.CategoryFilter.prototype.draw = function(dataTable, opt_options, opt_state) {};
google.visualization.CategoryFilter.prototype.applyFilter = function() {};
google.visualization.CategoryFilter.prototype.getState = function() {};
google.visualization.CategoryFilter.prototype.resetControl = function() {};
/** @constructor */
google.visualization.ChartRangeFilter = function(container) {};
google.visualization.ChartRangeFilter.prototype.draw = function(dataTable, opt_options, opt_state) {};
google.visualization.ChartRangeFilter.prototype.applyFilter = function() {};
google.visualization.ChartRangeFilter.prototype.getState = function() {};
google.visualization.ChartRangeFilter.prototype.resetControl = function() {};

3
dspace-xmlui/dspace-xmlui-webapp/src/main/webapp/static/js/statisticsElasticSearch/jquery.ui.datepicker-accessible.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

385
dspace-xmlui/dspace-xmlui-webapp/src/main/webapp/static/js/statisticsElasticSearch/visualizeData.js Normal file
View File

@@ -0,0 +1,385 @@
// # Visualize Data
//
// Simple script to help visualize data generated by elastic search.
// Firstly we wrap our code in a closure to keep variables local.
(function (context) {
(function ($) {
// ## Add jQuery datepicker functionality to the ReportGenerator
$(".date-picker").datepicker({dateFormat: "mm/dd/yy"});
var dateStart = new Date($('input[name=dateStart]').val());
var dateEnd = new Date($('input[name=dateEnd]').val());
// ### Chart Maker
//
// Create a helper module called chart maker that allows us to specify and
// draw charts.
context.ChartMaker = function() {
var chartMaker = {};
// A place to store charts to draw later.
chartMaker.charts = {};
// A shortcut to the google.visualization.DataTable function.
chartMaker.chartData = function () {
return new google.visualization.DataTable();
};
// The `addChart` function is used to add chart data to `ChartMaker`.
// It does two things: Creates a div under the specified parent div
// to put the chart in and add a new chart to `ChartMaker`'s internal
// list of charts.
//
// `addChart` takes an object (`userConfig`) that should have the
// following keys.
//
// * `entries` {object} The entries we use to create the chart.
// * `name` {string} A nice name to give the chart so the user can reference it
// later.
// * `includeTotal` {boolean} Whether or not to include the total in
// each row.
// * `chartData` {object} An instance of ChartMaker.chartData containing the data for
// the chart being added.
// * `keyField` {string} The name of the key to use on entries.
// * `valueField` {string} The name of the value to use on entries.
// * `parentElement` {string} The parent div to create the chart under
// * `chartType` {string} The key specifying what type of chart to
// use from `google.visualization`.
chartMaker.addChart = function (userConfig) {
var c = $.extend({
entries: [],
name: '',
includeTotal: false,
chartData: null,
dataSection: null,
keyField: 'term',
valueField: 'count',
parentElement: 'aspect_statisticsElasticSearch_ElasticSearchStatsViewer_div_chart_div',
chartType: 'GeoChart'
}, userConfig);
// `dataValue` will eventually become the rows of data in our
// chart.
var dataValue = [];
var total = 0;
// Cheat with dates / data, and zero-fill the start/end edges, and "hopefully" things work out...
// Set certainty to our cheat date to false, so it gets a dotted line.
if (c.chartData.getColumnType(0) == 'date' && isValidDate(dateStart) && c.chartType == 'LineChart') {
var cheatDateStart = [];
cheatDateStart.push(dateStart);
cheatDateStart.push(0);
if(c.includeTotal) {
cheatDateStart.push(total);
}
cheatDateStart.push(false);
dataValue.push(cheatDateStart);
}
// For each entry construct a vector to add to push onto
// `dataValue`.
$.each(c.entries, function(index, entry) {
if(c.dataSection != null) {
entry = entry[c.dataSection];
}
newEntry = [];
if (c.chartData.getColumnType(0) == 'date') {
newEntry.push(new Date(entry[c.keyField]));
} else {
newEntry.push(entry[c.keyField]);
}
newEntry.push(entry[c.valueField]);
if (c.includeTotal) {
total += entry[c.valueField];
newEntry.push(total);
}
// Certain data gets a certainty score of true. (solid line)
if (c.chartData.getColumnType(0) == 'date' && c.chartType=='LineChart') {
newEntry.push(true);
}
dataValue.push(newEntry);
});
//Cheat and zero-fill in the last date. Certainty is false, so dotted line.
if (c.chartData.getColumnType(0) == 'date' && isValidDate(dateEnd) && c.chartType=='LineChart') {
var cheatDateEnd = [];
cheatDateEnd.push(dateEnd);
cheatDateEnd.push(0);
if(c.includeTotal) {
cheatDateEnd.push(total);
}
cheatDateEnd.push(false);
dataValue.push(cheatDateEnd);
}
// Add rows (`dataValue`) to the chartData.
c.chartData.addRows(dataValue);
// Add a child element
var par = $('#' + c.parentElement);
par.append("<div style='height:280px; width:675px;' " +
"id='dspaceChart_" + c.name + "'> </div>");
this.charts[c.name] = {
chart: new google.visualization[c.chartType](
document.getElementById("dspaceChart_" + c.name)),
data: c.chartData,
options: c.options
};
};
// `drawChart` takes a chart from ChartMaker's internal `charts` array
// (specified by the `name` parameter) and draws that chart.
chartMaker.drawChart = function(name, globalOptions) {
if (typeof globalOptions === 'undefined') {
globalOptions = {};
}
var cobj = this.charts[name];
// Allow the user to overwrite data with a passed in options.
var data = cobj.data;
if ('data' in globalOptions) {
data = globalOptions.data;
}
// Merge the Global Options with the local options for the chart
var combinedOptions = $.extend(globalOptions, cobj.options);
// Draw the named chart!
cobj.chart.draw(data, combinedOptions);
};
// `drawAllCharts` simply loops through the defined charts and draws
// each one.
chartMaker.drawAllCharts = function (options) {
for (var name in this.charts) {
this.drawChart(name, options);
}
};
return chartMaker;
};
})(jQuery);
// ## Now some user level code. . .
// Load the visualization Library
google.load('visualization', '1',{'packages':['annotatedtimeline', 'geochart', 'corechart', 'table']});
// Set the callback for once the visualization library has loaded and make
// sure the DOM has loaded as well.
google.setOnLoadCallback(function () {
jQuery(document).ready(function ($) {
//Create a ChartMaker instance.
var chartMaker = new ChartMaker();
// Get data from elastic that has been dumped on the page.
var elasticJSON = $.parseJSON($('#aspect_statisticsElasticSearch_ElasticSearchStatsViewer_field_response').val());
// `function chartDataHelper` creates a chartData object from a few
// parameters.
// Required userConfig values: textKey, textValue
// Optional userConfig values: textTotal, hasCertainty
function chartDataHelper(userConfig) {
// Set some defaults, but allow user over-ride
var c = $.extend({
type:'string',
includeTotal : false,
hasCertainty: false
}, userConfig);
// Put data from Elastic response into a ChartData object
var main_chart_data = chartMaker.chartData();
main_chart_data.addColumn(c.type, c.textKey);
main_chart_data.addColumn('number', c.textValue);
if (c.includeTotal) {
main_chart_data.addColumn('number', c.textTotal);
}
// Add a certainty column for date data.
if (c.hasCertainty == true) {
main_chart_data.addColumn({type:'boolean',role:'certainty'}); // certainty col.
}
return main_chart_data;
}
// Set the title for the charts.
var options = { title : 'Views per DSpaceObject Type' };
// ### Start adding charts!
//
// Use a helper to do all the work to create the
// associated charts data tables.
// There is one parent div chart_div, and we will append child divs for each chart.
// Add a chart to show total downloads.
var name = $('input[name=containerName]').val();
var optionsDownloads = {title: 'Number of File Downloads: ' + name };
// Add a chart to show monthly downloads (without the total).
if ((elasticJSON !== null) && (typeof elasticJSON.facets.monthly_downloads !== 'undefined')) {
var chartDataNoTotal = chartDataHelper({
type : 'date',
textKey : 'Date',
textValue : 'File Downloads',
hasCertainty: true
});
chartMaker.addChart({
entries: elasticJSON.facets.monthly_downloads.entries,
name: 'downloadsMonthly',
chartData: chartDataNoTotal,
keyField: 'time',
chartType: 'LineChart',
options: optionsDownloads});
if ($('input[name=reportDepth]').val() == "detail") {
var chartDataTotal = chartDataHelper({
type : 'date',
textKey : 'Date',
textValue : 'File Downloads',
includeTotal: true,
textTotal: 'Total Downloads'
});
// Table with raw data of # Downloads each month
chartMaker.addChart({
entries: elasticJSON.facets.monthly_downloads.entries,
name: 'downloadsMonthlyTable',
chartData: chartDataTotal,
includeTotal : true,
keyField : 'time',
options:optionsDownloads,
chartType: 'Table'
});
// Chart of Downloads with aggregate total
var chartDataTotal2 = chartDataHelper({
type : 'date',
textKey : 'Date',
textValue : 'File Downloads',
includeTotal: true,
textTotal: 'Total Downloads',
hasCertainty: true
});
chartMaker.addChart({
entries: elasticJSON.facets.monthly_downloads.entries,
name: 'downloadsWithTotal',
includeTotal: true,
chartData: chartDataTotal2,
keyField: 'time',
chartType: 'LineChart',
options: optionsDownloads});
}
}
// Add a chart to show downloads from various countries.
if ((elasticJSON !== null) && (typeof elasticJSON.facets.top_countries !== 'undefined')) {
var chartDataGeo = chartDataHelper({
type : 'string',
textKey : 'Country',
textValue : 'Downloads'
});
chartMaker.addChart({
entries: elasticJSON.facets.top_countries.terms,
name: 'topCountries',
chartData: chartDataGeo,
options: options});
if ($('input[name=reportDepth]').val() == "detail") {
chartMaker.addChart({
entries: elasticJSON.facets.top_countries.terms,
name: 'topCountriesTable',
chartData: chartDataGeo,
options:options,
chartType: 'Table'
});
}
}
// Add a chart to show downloads from various countries.
if ((elasticJSON !== null) && typeof elasticJSON.facets.top_US_cities !== 'undefined' && $('input[name=reportDepth]').val() == "detail") {
var chartDataGeoUS = chartDataHelper({
type : 'string',
textKey : 'City',
textValue : 'Downloads'
});
var optionsUS = {region : 'US', displayMode : 'markers', resolution : 'provinces', magnifyingGlass : {enable: true, zoomFactor: 7.5} };
chartMaker.addChart({
entries: elasticJSON.facets.top_US_cities.terms,
name: 'topUSCities',
chartData: chartDataGeoUS,
options: optionsUS});
}
// Add a pie chart that shows top DSO Types usage.
/*
if (typeof elasticJSON.facets.top_types !== 'undefined') {
var chartDataPie = chartDataHelper('string', 'Type', 'Views', false, '');
chartMaker.addChart({
entries: elasticJSON.facets.top_types.terms,
name: 'topTypes',
chartData: chartDataPie,
chartType: 'PieChart',
options: options});
}
*/
// Finally, we draw all of the charts.
chartMaker.drawAllCharts();
//Set Titles to Charts that cannot otherwise set titles automatically (geocharts).
var baseURLStats = $('input[name=baseURLStats]').val();
var timeRangeString = $('input[name=timeRangeString]').val();
//TODO these dates are already accessed in different scope/context, its a waste to reexecute
var fromDateString = $('input[name=dateStart]').val();
var toDateString = $('input[name=dateEnd]').val();
if ($('input[name=reportDepth]').val() == "summary") {
$('<p>'+timeRangeString+' <a href="'+ baseURLStats + '/itemsAdded">For more information.</a></p>').insertBefore('#aspect_statisticsElasticSearch_ElasticSearchStatsViewer_table_itemsAddedGrid');
$('<p>'+timeRangeString+' <a href="'+ baseURLStats + '/filesAdded">For more information.</a></p>').insertBefore('#aspect_statisticsElasticSearch_ElasticSearchStatsViewer_table_filesInContainer-grid');
$('<h3>Number of File Downloads for ' + name + '</h3>'+timeRangeString+' <a href="'+ baseURLStats + '/fileDownloads">For more information.</a>').insertBefore('#dspaceChart_downloadsMonthly');
$('<h3>Countries with most Downloads ' + name + '</h3>'+timeRangeString+' <a href="'+ baseURLStats + '/topCountries">For more information.</a>').insertBefore('#dspaceChart_topCountries');
$('<p>'+timeRangeString+' <a href="'+ baseURLStats + '/topUSCities">For more information.</a></p>').insertBefore('#dspaceChart_topUSCities');
$('<p>'+timeRangeString+' <a href="'+ baseURLStats + '/topDownloads">For more information.</a></p>').insertBefore('#aspect_statisticsElasticSearch_ElasticSearchStatsViewer_table_facet-Bitstream');
}
var reportName = $('input[name=reportName]').val();
if ($('input[name=reportDepth]').val() == "detail") {
var contextPanel = '<div><p><a href="' + baseURLStats + '">Back to Summary Statistics for ' + name + '</a></p><br/>';
contextPanel += '<a href="#" onclick="window.print(); return false;"><img src="http://www.famfamfam.com/lab/icons/silk/icons/printer.png"/>Print This Report</a><br/>';
contextPanel += '<a href="' + baseURLStats + '/csv/' + reportName;
if(fromDateString !== null && typeof fromDateString !== 'undefined') {
contextPanel += '?from=' + fromDateString;
}
if(toDateString !== null && typeof toDateString !== 'undefined') {
if(fromDateString !== null && typeof fromDateString !== 'undefined') {
contextPanel += '&';
} else {
contextPanel =+ '?';
}
contextPanel += 'to=' + toDateString;
}
contextPanel += '"><img src="http://www.famfamfam.com/lab/icons/silk/icons/page_excel.png"/>Download Data as .csv</a></div>';
$(contextPanel).insertAfter('#aspect_statisticsElasticSearch_ElasticSearchStatsViewer_div_chart_div');
}
});
});
})(this);
function isValidDate(d) {
if ( Object.prototype.toString.call(d) !== "[object Date]" )
return false;
return !isNaN(d.getTime());
}

7
dspace-xmlui/dspace-xmlui-webapp/src/main/webapp/themes/Mirage/lib/xsl/core/page-structure.xsl
View File

@@ -641,6 +641,13 @@
<!-- Add theme javascipt -->
<xsl:for-each select="/dri:document/dri:meta/dri:pageMeta/dri:metadata[@element='javascript'][@qualifier='url']">
<script type="text/javascript">
<xsl:attribute name="src">
<xsl:value-of select="."/>
</xsl:attribute>&#160;</script>
</xsl:for-each>
<xsl:for-each select="/dri:document/dri:meta/dri:pageMeta/dri:metadata[@element='javascript'][not(@qualifier)]">
<script type="text/javascript">
<xsl:attribute name="src">

7
dspace-xmlui/dspace-xmlui-webapp/src/main/webapp/themes/dri2xhtml-alt/core/page-structure.xsl
View File

@@ -196,6 +196,13 @@
</script>
<!-- Add theme javascipt -->
<xsl:for-each select="/dri:document/dri:meta/dri:pageMeta/dri:metadata[@element='javascript'][@qualifier='url']">
<script type="text/javascript">
<xsl:attribute name="src">
<xsl:value-of select="."/>
</xsl:attribute>&#160;</script>
</xsl:for-each>
<xsl:for-each select="/dri:document/dri:meta/dri:pageMeta/dri:metadata[@element='javascript'][not(@qualifier)]">
<script type="text/javascript">
<xsl:attribute name="src">

30
dspace-xmlui/dspace-xmlui-webapp/src/main/webapp/themes/dri2xhtml/structural.xsl
View File

@@ -229,7 +229,25 @@
return true;
}
</script>
<!-- add "shared" javascript from static, path is relative to webapp root -->
<xsl:for-each select="/dri:document/dri:meta/dri:pageMeta/dri:metadata[@element='javascript'][@qualifier='url']">
<script type="text/javascript">
<xsl:attribute name="src">
<xsl:value-of select="."/>
</xsl:attribute>&#160;</script>
</xsl:for-each>
<!-- add "shared" javascript from static, path is relative to webapp root -->
<xsl:for-each select="/dri:document/dri:meta/dri:pageMeta/dri:metadata[@element='javascript'][@qualifier='static']">
<script type="text/javascript">
<xsl:attribute name="src">
<xsl:value-of select="/dri:document/dri:meta/dri:pageMeta/dri:metadata[@element='contextPath'][not(@qualifier)]"/>
<xsl:text>/</xsl:text>
<xsl:value-of select="."/>
</xsl:attribute>&#160;</script>
</xsl:for-each>
<!-- Add theme javascipt -->
<xsl:for-each select="/dri:document/dri:meta/dri:pageMeta/dri:metadata[@element='javascript'][not(@qualifier)]">
<script type="text/javascript">
@@ -242,16 +260,6 @@
</xsl:attribute>&#160;</script>
</xsl:for-each>
<!-- add "shared" javascript from static, path is relative to webapp root -->
<xsl:for-each select="/dri:document/dri:meta/dri:pageMeta/dri:metadata[@element='javascript'][@qualifier='static']">
<script type="text/javascript">
<xsl:attribute name="src">
<xsl:value-of select="/dri:document/dri:meta/dri:pageMeta/dri:metadata[@element='contextPath'][not(@qualifier)]"/>
<xsl:text>/</xsl:text>
<xsl:value-of select="."/>
</xsl:attribute>&#160;</script>
</xsl:for-each>
<!-- Add a google analytics script if the key is present -->
<xsl:if test="/dri:document/dri:meta/dri:pageMeta/dri:metadata[@element='google'][@qualifier='analytics']">

2
dspace/config/dspace.cfg
View File

@@ -220,7 +220,7 @@ log.dir = ${dspace.dir}/log
# If enabled, the logging and the solr statistics system will look for
# an X-Forward header. If it finds it, it will use this for the user IP address
#log.useProxies = true
#useProxies = true
##### Search settings #####

8
dspace/config/launcher.xml
View File

@@ -291,6 +291,14 @@
</step>
</command>
<command>
<name>stats-log-importer-elasticsearch</name>
<description>Import solr-format converted log files into Elastic Search statistics</description>
<step>
<class>org.dspace.statistics.util.StatisticsImporterElasticSearch</class>
</step>
</command>
<command>
<name>stats-util</name>
<description>Statistics Client for Maintenance of Solr Statistics Indexes</description>

8
dspace/config/modules/elastic-search-statistics.cfg Normal file
View File

@@ -0,0 +1,8 @@
# Elastic Search Engine for UsageEvent Statistics
#clusterName = dspacestatslogging
#indexName = dspaceindex
#indexType = stats
## Elastic Search can connect via TransportClient, for external ES service.
#address = 127.0.0.1
#statistics.elasticsearch.port = 9300

24
dspace/config/modules/solr-statistics.cfg
View File

@@ -11,33 +11,9 @@
# tomcat still running on port 8080
server = ${solr.server}/statistics
# The location for the Geo Database retrieved on update/installation
dbfile = ${dspace.dir}/config/GeoLiteCity.dat
# Timeout for the resolver in the dns lookup
# Time in milliseconds, defaults to 200 for backward compatibility
# Your systems default is usually set in /etc/resolv.conf and varies
# between 2 to 5 seconds, to high a value might result in solr exhausting
# your connection pool
resolver.timeout = 200
# Control if the statistics pages should be only shown to authorized users
# If enabled, only the administrators for the DSpaceObject will be able to
# view the statistics.
# If disabled, anyone with READ permissions on the DSpaceObject will be able
# to view the statistics.
authorization.admin=true
# A comma-separated list that contains the bundles for which the bitstreams will be displayed
query.filter.bundles=ORIGINAL
# Enable/disable logging of spiders in solr statistics.
# If false, and IP matches an address in spiderips.urls, event is not logged.
# If true, event will be logged with the 'isBot' field set to true
# (see query.filter.* for query filter options)
# Default value is true.
#logBots = true
# control solr statistics querying to filter out spider IPs
# false by default
#query.filter.spiderIp = false

23
dspace/config/modules/usage-statistics.cfg Normal file
View File

@@ -0,0 +1,23 @@
# The location for the Geo Database retrieved on update/installation
dbfile = ${dspace.dir}/config/GeoLiteCity.dat
# Timeout for the resolver in the dns lookup
# Time in milliseconds, defaults to 200 for backward compatibility
# Your systems default is usually set in /etc/resolv.conf and varies
# between 2 to 5 seconds, to high a value might result in solr exhausting
# your connection pool
resolver.timeout = 200
# Control if the statistics pages should be only shown to authorized users
# If enabled, only the administrators for the DSpaceObject will be able to
# view the statistics.
# If disabled, anyone with READ permissions on the DSpaceObject will be able
# to view the statistics.
authorization.admin=true
# Enable/disable logging of spiders in solr statistics.
# If false, and IP matches an address in spiderips.urls, event is not logged.
# If true, event will be logged with the 'isBot' field set to true
# (see query.filter.* for query filter options)
# Default value is true.
#logBots = true

1
dspace/config/xmlui.xconf
View File

@@ -71,6 +71,7 @@
<aspect name="E-Person" path="resource://aspects/EPerson/" />
<aspect name="Submission and Workflow" path="resource://aspects/Submission/" />
<aspect name="Statistics" path="resource://aspects/Statistics/" />
<aspect name="Statistics - Elastic Search" path="resource://aspects/StatisticsElasticSearch/" />
<aspect name="Original Workflow" path="resource://aspects/Workflow/" />
<!--<aspect name="XMLWorkflow" path="resource://aspects/XMLWorkflow/" />-->
<!--

14
pom.xml
View File

@@ -24,6 +24,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<filters.file>build.properties</filters.file>
<lucene.version>3.5.0</lucene.version>
</properties>
<build>
@@ -632,12 +633,12 @@
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
<version>3.3.0</version>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers</artifactId>
<version>3.3.0</version>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.dspace</groupId>
@@ -1205,4 +1206,13 @@
<distributionManagement>
<!-- further distribution management is found upstream in the sonatype parent -->
</distributionManagement>
<repositories>
<repository>
<id>sonatype-releases</id>
<name>Sonatype Releases Repository</name>
<url>http://oss.sonatype.org/content/repositories/releases/</url>
</repository>
</repositories>
</project>