diff --git a/dspace-api/pom.xml b/dspace-api/pom.xml index 8874722381..f69f2d8b37 100644 --- a/dspace-api/pom.xml +++ b/dspace-api/pom.xml @@ -576,9 +576,9 @@ commons-configuration - com.maxmind.geoip - geoip-api - 1.3.0 + com.maxmind.geoip2 + geoip2 + 2.11.0 org.apache.ant diff --git a/dspace-api/src/main/java/org/dspace/statistics/SolrLoggerServiceImpl.java b/dspace-api/src/main/java/org/dspace/statistics/SolrLoggerServiceImpl.java index 30282a8b41..6602ea93bf 100644 --- a/dspace-api/src/main/java/org/dspace/statistics/SolrLoggerServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/statistics/SolrLoggerServiceImpl.java @@ -7,10 +7,37 @@ */ package org.dspace.statistics; +import java.io.File; +import java.io.FileFilter; +import java.io.FileNotFoundException; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.io.UnsupportedEncodingException; +import java.net.InetAddress; +import java.net.URLEncoder; +import java.sql.SQLException; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import javax.servlet.http.HttpServletRequest; + import au.com.bytecode.opencsv.CSVReader; import au.com.bytecode.opencsv.CSVWriter; -import com.maxmind.geoip.Location; -import com.maxmind.geoip.LookupService; +import com.maxmind.geoip2.DatabaseReader; +import com.maxmind.geoip2.exception.GeoIp2Exception; +import com.maxmind.geoip2.model.CityResponse; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; @@ -55,15 +82,6 @@ import org.dspace.usage.UsageWorkflowEvent; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; -import javax.servlet.http.HttpServletRequest; -import java.io.*; -import java.net.URLEncoder; -import java.sql.SQLException; -import java.text.DateFormat; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.*; - /** * Static holder for a HttpSolrClient connection pool to issue * usage logging events to Solr from DSpace libraries, and some static query @@ -85,7 +103,7 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea public static final String DATE_FORMAT_DCDATE = "yyyy-MM-dd'T'HH:mm:ss'Z'"; - protected LookupService locationService; + protected DatabaseReader locationService; protected boolean useProxies; @@ -142,23 +160,24 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea // Read in the file so we don't have to do it all the time //spiderIps = SpiderDetector.getSpiderIpAddresses(); - LookupService service = null; + DatabaseReader service = null; // Get the db file for the location - String dbfile = configurationService.getProperty("usage-statistics.dbfile"); - if (dbfile != null) - { - try - { - service = new LookupService(dbfile, - LookupService.GEOIP_STANDARD); - } - catch (FileNotFoundException fe) - { - log.error("The GeoLite Database file is missing (" + dbfile + ")! Solr Statistics cannot generate location based reports! Please see the DSpace installation instructions for instructions to install this file.", fe); - } - catch (IOException e) - { - log.error("Unable to load GeoLite Database file (" + dbfile + ")! You may need to reinstall it. See the DSpace installation instructions for more details.", e); + String dbPath = configurationService.getProperty("usage-statistics.dbfile"); + if (dbPath != null) { + try { + File dbFile = new File(dbPath); + service = new DatabaseReader.Builder(dbFile).build(); + } catch (FileNotFoundException fe) { + log.error( + "The GeoLite Database file is missing (" + dbPath + ")! Solr Statistics cannot generate location " + + "based reports! Please see the DSpace installation instructions for instructions to install " + + "this file.", + fe); + } catch (IOException e) { + log.error( + "Unable to load GeoLite Database file (" + dbPath + ")! You may need to reinstall it. See the " + + "DSpace installation instructions for more details.", + e); } } else @@ -315,30 +334,32 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea doc1.addField("isBot",isSpiderBot); // Save the location information if valid, save the event without // location information if not valid - if(locationService != null) - { - Location location = locationService.getLocation(ip); - if (location != null - && !("--".equals(location.countryCode) - && location.latitude == -180 && location.longitude == -180)) - { - try - { - doc1.addField("continent", LocationUtils - .getContinentCode(location.countryCode)); + if (locationService != null) { + try { + InetAddress ipAddress = InetAddress.getByName(ip); + CityResponse location = locationService.city(ipAddress); + String countryCode = location.getCountry().getIsoCode(); + double latitude = location.getLocation().getLatitude(); + double longitude = location.getLocation().getLongitude(); + if (!( + "--".equals(countryCode) + && latitude == -180 + && longitude == -180) + ) { + try { + doc1.addField("continent", LocationUtils + .getContinentCode(countryCode)); + } catch (Exception e) { + System.out + .println("COUNTRY ERROR: " + countryCode); + } + doc1.addField("countryCode", countryCode); + doc1.addField("city", location.getCity().getName()); + doc1.addField("latitude", latitude); + doc1.addField("longitude", longitude); } - catch (Exception e) - { - System.out - .println("COUNTRY ERROR: " + location.countryCode); - } - doc1.addField("countryCode", location.countryCode); - doc1.addField("city", location.city); - doc1.addField("latitude", location.latitude); - doc1.addField("longitude", location.longitude); - - - + } catch (IOException | GeoIp2Exception e) { + log.error("Unable to get location of request", e); } } } @@ -401,30 +422,32 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea doc1.addField("isBot",isSpiderBot); // Save the location information if valid, save the event without // location information if not valid - if(locationService != null) - { - Location location = locationService.getLocation(ip); - if (location != null - && !("--".equals(location.countryCode) - && location.latitude == -180 && location.longitude == -180)) - { - try - { - doc1.addField("continent", LocationUtils - .getContinentCode(location.countryCode)); + if (locationService != null) { + try { + InetAddress ipAddress = InetAddress.getByName(ip); + CityResponse location = locationService.city(ipAddress); + String countryCode = location.getCountry().getIsoCode(); + double latitude = location.getLocation().getLatitude(); + double longitude = location.getLocation().getLongitude(); + if (!( + "--".equals(countryCode) + && latitude == -180 + && longitude == -180) + ) { + try { + doc1.addField("continent", LocationUtils + .getContinentCode(countryCode)); + } catch (Exception e) { + System.out + .println("COUNTRY ERROR: " + countryCode); + } + doc1.addField("countryCode", countryCode); + doc1.addField("city", location.getCity().getName()); + doc1.addField("latitude", latitude); + doc1.addField("longitude", longitude); } - catch (Exception e) - { - System.out - .println("COUNTRY ERROR: " + location.countryCode); - } - doc1.addField("countryCode", location.countryCode); - doc1.addField("city", location.city); - doc1.addField("latitude", location.latitude); - doc1.addField("longitude", location.longitude); - - - + } catch (GeoIp2Exception | IOException e) { + log.error("Unable to get location of request", e); } } } diff --git a/dspace-api/src/main/java/org/dspace/statistics/util/StatisticsImporter.java b/dspace-api/src/main/java/org/dspace/statistics/util/StatisticsImporter.java index 237c54e048..f034619c7d 100644 --- a/dspace-api/src/main/java/org/dspace/statistics/util/StatisticsImporter.java +++ b/dspace-api/src/main/java/org/dspace/statistics/util/StatisticsImporter.java @@ -7,7 +7,32 @@ */ package org.dspace.statistics.util; -import org.apache.commons.cli.*; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.FilenameFilter; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.InetAddress; +import java.text.DateFormat; +import java.text.DecimalFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Random; +import java.util.UUID; + +import com.maxmind.geoip2.DatabaseReader; +import com.maxmind.geoip2.model.CityResponse; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; import org.apache.commons.lang.time.DateFormatUtils; import org.apache.log4j.Logger; import org.apache.solr.common.SolrInputDocument; @@ -22,13 +47,6 @@ import org.dspace.core.ConfigurationManager; import org.dspace.eperson.EPerson; import org.dspace.eperson.factory.EPersonServiceFactory; import org.dspace.statistics.SolrLoggerServiceImpl; - -import java.text.*; -import java.io.*; -import java.util.*; - -import com.maxmind.geoip.LookupService; -import com.maxmind.geoip.Location; import org.dspace.statistics.factory.StatisticsServiceFactory; import org.dspace.statistics.service.SolrLoggerService; @@ -55,8 +73,10 @@ public class StatisticsImporter /** Solr server connection */ private static HttpSolrServer solr; - /** GEOIP lookup service */ - private static LookupService geoipLookup; + /** + * GEOIP lookup service + */ + private static DatabaseReader geoipLookup; /** Whether to skip the DNS reverse lookup or not */ private static boolean skipReverseDNS = false; @@ -191,8 +211,8 @@ public class StatisticsImporter String continent = ""; String country = ""; String countryCode = ""; - float longitude = 0f; - float latitude = 0f; + double longitude = 0f; + double latitude = 0f; String city = ""; String dns; @@ -257,15 +277,15 @@ public class StatisticsImporter } // Get the geo information for the user - Location location; try { - location = geoipLookup.getLocation(ip); - city = location.city; - country = location.countryName; - countryCode = location.countryCode; - longitude = location.longitude; - latitude = location.latitude; - if(verbose) { + InetAddress ipAddress = InetAddress.getByName(ip); + CityResponse cityResponse = geoipLookup.city(ipAddress); + city = cityResponse.getCity().getName(); + country = cityResponse.getCountry().getName(); + countryCode = cityResponse.getCountry().getIsoCode(); + longitude = cityResponse.getLocation().getLongitude(); + latitude = cityResponse.getLocation().getLatitude(); + if (verbose) { data += (", country = " + country); data += (", city = " + city); System.out.println(data); @@ -476,18 +496,21 @@ public class StatisticsImporter } solr = new HttpSolrServer(sserver); - String dbfile = ConfigurationManager.getProperty("usage-statistics", "dbfile"); - try - { - geoipLookup = new LookupService(dbfile, LookupService.GEOIP_STANDARD); - } - catch (FileNotFoundException fe) - { - log.error("The GeoLite Database file is missing (" + dbfile + ")! Solr Statistics cannot generate location based reports! Please see the DSpace installation instructions for instructions to install this file.", fe); - } - catch (IOException e) - { - log.error("Unable to load GeoLite Database file (" + dbfile + ")! You may need to reinstall it. See the DSpace installation instructions for more details.", e); + String dbPath = ConfigurationManager.getProperty("usage-statistics", "dbfile"); + try { + File dbFile = new File(dbPath); + geoipLookup = new DatabaseReader.Builder(dbFile).build(); + } catch (FileNotFoundException fe) { + log.error( + "The GeoLite Database file is missing (" + dbPath + ")! Solr Statistics cannot generate location " + + "based reports! Please see the DSpace installation instructions for instructions to install this " + + "file.", + fe); + } catch (IOException e) { + log.error( + "Unable to load GeoLite Database file (" + dbPath + ")! You may need to reinstall it. See the DSpace " + + "installation instructions for more details.", + e); }