hazza/DSpace
1
0
Fork 0
mirror of https://github.com/DSpace/DSpace.git synced 2025-10-07 10:04:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

[DS-3832] Upgrade to GeoIP2.

Browse Source
This commit is contained in:
Mark H. Wood
2018-03-21 15:49:38 -04:00
parent 0e7c7c0886
commit 4ea7575e4a
3 changed files with 155 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
dspace-api/pom.xml
View File

@@ -576,9 +576,9 @@
<artifactId>commons-configuration</artifactId> <artifactId>commons-configuration</artifactId>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.maxmind.geoip</groupId> <groupId>com.maxmind.geoip2</groupId>
<artifactId>geoip-api</artifactId> <artifactId>geoip2</artifactId>
<version>1.3.0</version> <version>2.11.0</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.ant</groupId> <groupId>org.apache.ant</groupId>

171
dspace-api/src/main/java/org/dspace/statistics/SolrLoggerServiceImpl.java
View File

@@ -7,10 +7,37 @@
*/ */
package org.dspace.statistics; package org.dspace.statistics;
import java.io.File;
import java.io.FileFilter;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.net.InetAddress;
import java.net.URLEncoder;
import java.sql.SQLException;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.servlet.http.HttpServletRequest;
import au.com.bytecode.opencsv.CSVReader; import au.com.bytecode.opencsv.CSVReader;
import au.com.bytecode.opencsv.CSVWriter; import au.com.bytecode.opencsv.CSVWriter;
import com.maxmind.geoip.Location; import com.maxmind.geoip2.DatabaseReader;
import com.maxmind.geoip.LookupService; import com.maxmind.geoip2.exception.GeoIp2Exception;
import com.maxmind.geoip2.model.CityResponse;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
@@ -55,15 +82,6 @@ import org.dspace.usage.UsageWorkflowEvent;
import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import javax.servlet.http.HttpServletRequest;
import java.io.*;
import java.net.URLEncoder;
import java.sql.SQLException;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
/** /**
* Static holder for a HttpSolrClient connection pool to issue * Static holder for a HttpSolrClient connection pool to issue
* usage logging events to Solr from DSpace libraries, and some static query * usage logging events to Solr from DSpace libraries, and some static query
@@ -85,7 +103,7 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea
public static final String DATE_FORMAT_DCDATE = "yyyy-MM-dd'T'HH:mm:ss'Z'"; public static final String DATE_FORMAT_DCDATE = "yyyy-MM-dd'T'HH:mm:ss'Z'";
protected LookupService locationService; protected DatabaseReader locationService;
protected boolean useProxies; protected boolean useProxies;
@@ -142,23 +160,24 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea
// Read in the file so we don't have to do it all the time // Read in the file so we don't have to do it all the time
//spiderIps = SpiderDetector.getSpiderIpAddresses(); //spiderIps = SpiderDetector.getSpiderIpAddresses();
LookupService service = null; DatabaseReader service = null;
// Get the db file for the location // Get the db file for the location
String dbfile = configurationService.getProperty("usage-statistics.dbfile"); String dbPath = configurationService.getProperty("usage-statistics.dbfile");
if (dbfile != null) if (dbPath != null) {
{ try {
try File dbFile = new File(dbPath);
{ service = new DatabaseReader.Builder(dbFile).build();
service = new LookupService(dbfile, } catch (FileNotFoundException fe) {
LookupService.GEOIP_STANDARD); log.error(
} "The GeoLite Database file is missing (" + dbPath + ")! Solr Statistics cannot generate location " +
catch (FileNotFoundException fe) "based reports! Please see the DSpace installation instructions for instructions to install " +
{ "this file.",
log.error("The GeoLite Database file is missing (" + dbfile + ")! Solr Statistics cannot generate location based reports! Please see the DSpace installation instructions for instructions to install this file.", fe); fe);
} } catch (IOException e) {
catch (IOException e) log.error(
{ "Unable to load GeoLite Database file (" + dbPath + ")! You may need to reinstall it. See the " +
log.error("Unable to load GeoLite Database file (" + dbfile + ")! You may need to reinstall it. See the DSpace installation instructions for more details.", e); "DSpace installation instructions for more details.",
e);
} }
} }
else else
@@ -315,30 +334,32 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea
doc1.addField("isBot",isSpiderBot); doc1.addField("isBot",isSpiderBot);
// Save the location information if valid, save the event without // Save the location information if valid, save the event without
// location information if not valid // location information if not valid
if(locationService != null) if (locationService != null) {
{ try {
Location location = locationService.getLocation(ip); InetAddress ipAddress = InetAddress.getByName(ip);
if (location != null CityResponse location = locationService.city(ipAddress);
&& !("--".equals(location.countryCode) String countryCode = location.getCountry().getIsoCode();
&& location.latitude == -180 && location.longitude == -180)) double latitude = location.getLocation().getLatitude();
{ double longitude = location.getLocation().getLongitude();
try if (!(
{ "--".equals(countryCode)
doc1.addField("continent", LocationUtils && latitude == -180
.getContinentCode(location.countryCode)); && longitude == -180)
) {
try {
doc1.addField("continent", LocationUtils
.getContinentCode(countryCode));
} catch (Exception e) {
System.out
.println("COUNTRY ERROR: " + countryCode);
}
doc1.addField("countryCode", countryCode);
doc1.addField("city", location.getCity().getName());
doc1.addField("latitude", latitude);
doc1.addField("longitude", longitude);
} }
catch (Exception e) } catch (IOException | GeoIp2Exception e) {
{ log.error("Unable to get location of request", e);
System.out
.println("COUNTRY ERROR: " + location.countryCode);
}
doc1.addField("countryCode", location.countryCode);
doc1.addField("city", location.city);
doc1.addField("latitude", location.latitude);
doc1.addField("longitude", location.longitude);
} }
} }
} }
@@ -401,30 +422,32 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea
doc1.addField("isBot",isSpiderBot); doc1.addField("isBot",isSpiderBot);
// Save the location information if valid, save the event without // Save the location information if valid, save the event without
// location information if not valid // location information if not valid
if(locationService != null) if (locationService != null) {
{ try {
Location location = locationService.getLocation(ip); InetAddress ipAddress = InetAddress.getByName(ip);
if (location != null CityResponse location = locationService.city(ipAddress);
&& !("--".equals(location.countryCode) String countryCode = location.getCountry().getIsoCode();
&& location.latitude == -180 && location.longitude == -180)) double latitude = location.getLocation().getLatitude();
{ double longitude = location.getLocation().getLongitude();
try if (!(
{ "--".equals(countryCode)
doc1.addField("continent", LocationUtils && latitude == -180
.getContinentCode(location.countryCode)); && longitude == -180)
) {
try {
doc1.addField("continent", LocationUtils
.getContinentCode(countryCode));
} catch (Exception e) {
System.out
.println("COUNTRY ERROR: " + countryCode);
}
doc1.addField("countryCode", countryCode);
doc1.addField("city", location.getCity().getName());
doc1.addField("latitude", latitude);
doc1.addField("longitude", longitude);
} }
catch (Exception e) } catch (GeoIp2Exception | IOException e) {
{ log.error("Unable to get location of request", e);
System.out
.println("COUNTRY ERROR: " + location.countryCode);
}
doc1.addField("countryCode", location.countryCode);
doc1.addField("city", location.city);
doc1.addField("latitude", location.latitude);
doc1.addField("longitude", location.longitude);
} }
} }
} }

87
dspace-api/src/main/java/org/dspace/statistics/util/StatisticsImporter.java
View File

@@ -7,7 +7,32 @@
*/ */
package org.dspace.statistics.util; package org.dspace.statistics.util;
import org.apache.commons.cli.*; import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.InetAddress;
import java.text.DateFormat;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Random;
import java.util.UUID;
import com.maxmind.geoip2.DatabaseReader;
import com.maxmind.geoip2.model.CityResponse;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.PosixParser;
import org.apache.commons.lang.time.DateFormatUtils; import org.apache.commons.lang.time.DateFormatUtils;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.SolrInputDocument;
@@ -22,13 +47,6 @@ import org.dspace.core.ConfigurationManager;
import org.dspace.eperson.EPerson; import org.dspace.eperson.EPerson;
import org.dspace.eperson.factory.EPersonServiceFactory; import org.dspace.eperson.factory.EPersonServiceFactory;
import org.dspace.statistics.SolrLoggerServiceImpl; import org.dspace.statistics.SolrLoggerServiceImpl;
import java.text.*;
import java.io.*;
import java.util.*;
import com.maxmind.geoip.LookupService;
import com.maxmind.geoip.Location;
import org.dspace.statistics.factory.StatisticsServiceFactory; import org.dspace.statistics.factory.StatisticsServiceFactory;
import org.dspace.statistics.service.SolrLoggerService; import org.dspace.statistics.service.SolrLoggerService;
@@ -55,8 +73,10 @@ public class StatisticsImporter
/** Solr server connection */ /** Solr server connection */
private static HttpSolrServer solr; private static HttpSolrServer solr;
/** GEOIP lookup service */ /**
private static LookupService geoipLookup; * GEOIP lookup service
*/
private static DatabaseReader geoipLookup;
/** Whether to skip the DNS reverse lookup or not */ /** Whether to skip the DNS reverse lookup or not */
private static boolean skipReverseDNS = false; private static boolean skipReverseDNS = false;
@@ -191,8 +211,8 @@ public class StatisticsImporter
String continent = ""; String continent = "";
String country = ""; String country = "";
String countryCode = ""; String countryCode = "";
float longitude = 0f; double longitude = 0f;
float latitude = 0f; double latitude = 0f;
String city = ""; String city = "";
String dns; String dns;
@@ -257,15 +277,15 @@ public class StatisticsImporter
} }
// Get the geo information for the user // Get the geo information for the user
Location location;
try { try {
location = geoipLookup.getLocation(ip); InetAddress ipAddress = InetAddress.getByName(ip);
city = location.city; CityResponse cityResponse = geoipLookup.city(ipAddress);
country = location.countryName; city = cityResponse.getCity().getName();
countryCode = location.countryCode; country = cityResponse.getCountry().getName();
longitude = location.longitude; countryCode = cityResponse.getCountry().getIsoCode();
latitude = location.latitude; longitude = cityResponse.getLocation().getLongitude();
if(verbose) { latitude = cityResponse.getLocation().getLatitude();
if (verbose) {
data += (", country = " + country); data += (", country = " + country);
data += (", city = " + city); data += (", city = " + city);
System.out.println(data); System.out.println(data);
@@ -476,18 +496,21 @@ public class StatisticsImporter
} }
solr = new HttpSolrServer(sserver); solr = new HttpSolrServer(sserver);
String dbfile = ConfigurationManager.getProperty("usage-statistics", "dbfile"); String dbPath = ConfigurationManager.getProperty("usage-statistics", "dbfile");
try try {
{ File dbFile = new File(dbPath);
geoipLookup = new LookupService(dbfile, LookupService.GEOIP_STANDARD); geoipLookup = new DatabaseReader.Builder(dbFile).build();
} } catch (FileNotFoundException fe) {
catch (FileNotFoundException fe) log.error(
{ "The GeoLite Database file is missing (" + dbPath + ")! Solr Statistics cannot generate location " +
log.error("The GeoLite Database file is missing (" + dbfile + ")! Solr Statistics cannot generate location based reports! Please see the DSpace installation instructions for instructions to install this file.", fe); "based reports! Please see the DSpace installation instructions for instructions to install this " +
} "file.",
catch (IOException e) fe);
{ } catch (IOException e) {
log.error("Unable to load GeoLite Database file (" + dbfile + ")! You may need to reinstall it. See the DSpace installation instructions for more details.", e); log.error(
"Unable to load GeoLite Database file (" + dbPath + ")! You may need to reinstall it. See the DSpace " +
"installation instructions for more details.",
e);
} }