hazza/DSpace
1
0
Fork 0
mirror of https://github.com/DSpace/DSpace.git synced 2025-10-07 10:04:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

[DS-3832] Upgrade to GeoIP2.

Browse Source
This commit is contained in:
Mark H. Wood
2018-03-21 15:49:38 -04:00
parent 0e7c7c0886
commit 4ea7575e4a
3 changed files with 155 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
dspace-api/pom.xml
View File

@@ -576,9 +576,9 @@
<artifactId>commons-configuration</artifactId>
</dependency>
<dependency>
<groupId>com.maxmind.geoip</groupId>
<artifactId>geoip-api</artifactId>
<version>1.3.0</version>
<groupId>com.maxmind.geoip2</groupId>
<artifactId>geoip2</artifactId>
<version>2.11.0</version>
</dependency>
<dependency>
<groupId>org.apache.ant</groupId>

163
dspace-api/src/main/java/org/dspace/statistics/SolrLoggerServiceImpl.java
View File

@@ -7,10 +7,37 @@
*/
package org.dspace.statistics;
import java.io.File;
import java.io.FileFilter;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.net.InetAddress;
import java.net.URLEncoder;
import java.sql.SQLException;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.servlet.http.HttpServletRequest;
import au.com.bytecode.opencsv.CSVReader;
import au.com.bytecode.opencsv.CSVWriter;
import com.maxmind.geoip.Location;
import com.maxmind.geoip.LookupService;
import com.maxmind.geoip2.DatabaseReader;
import com.maxmind.geoip2.exception.GeoIp2Exception;
import com.maxmind.geoip2.model.CityResponse;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
@@ -55,15 +82,6 @@ import org.dspace.usage.UsageWorkflowEvent;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import javax.servlet.http.HttpServletRequest;
import java.io.*;
import java.net.URLEncoder;
import java.sql.SQLException;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
/**
* Static holder for a HttpSolrClient connection pool to issue
* usage logging events to Solr from DSpace libraries, and some static query
@@ -85,7 +103,7 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea
public static final String DATE_FORMAT_DCDATE = "yyyy-MM-dd'T'HH:mm:ss'Z'";
protected LookupService locationService;
protected DatabaseReader locationService;
protected boolean useProxies;
@@ -142,23 +160,24 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea
// Read in the file so we don't have to do it all the time
//spiderIps = SpiderDetector.getSpiderIpAddresses();
LookupService service = null;
DatabaseReader service = null;
// Get the db file for the location
String dbfile = configurationService.getProperty("usage-statistics.dbfile");
if (dbfile != null)
{
try
{
service = new LookupService(dbfile,
LookupService.GEOIP_STANDARD);
}
catch (FileNotFoundException fe)
{
log.error("The GeoLite Database file is missing (" + dbfile + ")! Solr Statistics cannot generate location based reports! Please see the DSpace installation instructions for instructions to install this file.", fe);
}
catch (IOException e)
{
log.error("Unable to load GeoLite Database file (" + dbfile + ")! You may need to reinstall it. See the DSpace installation instructions for more details.", e);
String dbPath = configurationService.getProperty("usage-statistics.dbfile");
if (dbPath != null) {
try {
File dbFile = new File(dbPath);
service = new DatabaseReader.Builder(dbFile).build();
} catch (FileNotFoundException fe) {
log.error(
"The GeoLite Database file is missing (" + dbPath + ")! Solr Statistics cannot generate location " +
"based reports! Please see the DSpace installation instructions for instructions to install " +
"this file.",
fe);
} catch (IOException e) {
log.error(
"Unable to load GeoLite Database file (" + dbPath + ")! You may need to reinstall it. See the " +
"DSpace installation instructions for more details.",
e);
}
}
else
@@ -315,30 +334,32 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea
doc1.addField("isBot",isSpiderBot);
// Save the location information if valid, save the event without
// location information if not valid
if(locationService != null)
{
Location location = locationService.getLocation(ip);
if (location != null
&& !("--".equals(location.countryCode)
&& location.latitude == -180 && location.longitude == -180))
{
try
{
if (locationService != null) {
try {
InetAddress ipAddress = InetAddress.getByName(ip);
CityResponse location = locationService.city(ipAddress);
String countryCode = location.getCountry().getIsoCode();
double latitude = location.getLocation().getLatitude();
double longitude = location.getLocation().getLongitude();
if (!(
"--".equals(countryCode)
&& latitude == -180
&& longitude == -180)
) {
try {
doc1.addField("continent", LocationUtils
.getContinentCode(location.countryCode));
}
catch (Exception e)
{
.getContinentCode(countryCode));
} catch (Exception e) {
System.out
.println("COUNTRY ERROR: " + location.countryCode);
.println("COUNTRY ERROR: " + countryCode);
}
doc1.addField("countryCode", location.countryCode);
doc1.addField("city", location.city);
doc1.addField("latitude", location.latitude);
doc1.addField("longitude", location.longitude);
doc1.addField("countryCode", countryCode);
doc1.addField("city", location.getCity().getName());
doc1.addField("latitude", latitude);
doc1.addField("longitude", longitude);
}
} catch (IOException | GeoIp2Exception e) {
log.error("Unable to get location of request", e);
}
}
}
@@ -401,30 +422,32 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea
doc1.addField("isBot",isSpiderBot);
// Save the location information if valid, save the event without
// location information if not valid
if(locationService != null)
{
Location location = locationService.getLocation(ip);
if (location != null
&& !("--".equals(location.countryCode)
&& location.latitude == -180 && location.longitude == -180))
{
try
{
if (locationService != null) {
try {
InetAddress ipAddress = InetAddress.getByName(ip);
CityResponse location = locationService.city(ipAddress);
String countryCode = location.getCountry().getIsoCode();
double latitude = location.getLocation().getLatitude();
double longitude = location.getLocation().getLongitude();
if (!(
"--".equals(countryCode)
&& latitude == -180
&& longitude == -180)
) {
try {
doc1.addField("continent", LocationUtils
.getContinentCode(location.countryCode));
}
catch (Exception e)
{
.getContinentCode(countryCode));
} catch (Exception e) {
System.out
.println("COUNTRY ERROR: " + location.countryCode);
.println("COUNTRY ERROR: " + countryCode);
}
doc1.addField("countryCode", location.countryCode);
doc1.addField("city", location.city);
doc1.addField("latitude", location.latitude);
doc1.addField("longitude", location.longitude);
doc1.addField("countryCode", countryCode);
doc1.addField("city", location.getCity().getName());
doc1.addField("latitude", latitude);
doc1.addField("longitude", longitude);
}
} catch (GeoIp2Exception | IOException e) {
log.error("Unable to get location of request", e);
}
}
}

85
dspace-api/src/main/java/org/dspace/statistics/util/StatisticsImporter.java
View File

@@ -7,7 +7,32 @@
*/
package org.dspace.statistics.util;
import org.apache.commons.cli.*;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.InetAddress;
import java.text.DateFormat;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Random;
import java.util.UUID;
import com.maxmind.geoip2.DatabaseReader;
import com.maxmind.geoip2.model.CityResponse;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.PosixParser;
import org.apache.commons.lang.time.DateFormatUtils;
import org.apache.log4j.Logger;
import org.apache.solr.common.SolrInputDocument;
@@ -22,13 +47,6 @@ import org.dspace.core.ConfigurationManager;
import org.dspace.eperson.EPerson;
import org.dspace.eperson.factory.EPersonServiceFactory;
import org.dspace.statistics.SolrLoggerServiceImpl;
import java.text.*;
import java.io.*;
import java.util.*;
import com.maxmind.geoip.LookupService;
import com.maxmind.geoip.Location;
import org.dspace.statistics.factory.StatisticsServiceFactory;
import org.dspace.statistics.service.SolrLoggerService;
@@ -55,8 +73,10 @@ public class StatisticsImporter
/** Solr server connection */
private static HttpSolrServer solr;
/** GEOIP lookup service */
private static LookupService geoipLookup;
/**
* GEOIP lookup service
*/
private static DatabaseReader geoipLookup;
/** Whether to skip the DNS reverse lookup or not */
private static boolean skipReverseDNS = false;
@@ -191,8 +211,8 @@ public class StatisticsImporter
String continent = "";
String country = "";
String countryCode = "";
float longitude = 0f;
float latitude = 0f;
double longitude = 0f;
double latitude = 0f;
String city = "";
String dns;
@@ -257,14 +277,14 @@ public class StatisticsImporter
}
// Get the geo information for the user
Location location;
try {
location = geoipLookup.getLocation(ip);
city = location.city;
country = location.countryName;
countryCode = location.countryCode;
longitude = location.longitude;
latitude = location.latitude;
InetAddress ipAddress = InetAddress.getByName(ip);
CityResponse cityResponse = geoipLookup.city(ipAddress);
city = cityResponse.getCity().getName();
country = cityResponse.getCountry().getName();
countryCode = cityResponse.getCountry().getIsoCode();
longitude = cityResponse.getLocation().getLongitude();
latitude = cityResponse.getLocation().getLatitude();
if (verbose) {
data += (", country = " + country);
data += (", city = " + city);
@@ -476,18 +496,21 @@ public class StatisticsImporter
}
solr = new HttpSolrServer(sserver);
String dbfile = ConfigurationManager.getProperty("usage-statistics", "dbfile");
try
{
geoipLookup = new LookupService(dbfile, LookupService.GEOIP_STANDARD);
}
catch (FileNotFoundException fe)
{
log.error("The GeoLite Database file is missing (" + dbfile + ")! Solr Statistics cannot generate location based reports! Please see the DSpace installation instructions for instructions to install this file.", fe);
}
catch (IOException e)
{
log.error("Unable to load GeoLite Database file (" + dbfile + ")! You may need to reinstall it. See the DSpace installation instructions for more details.", e);
String dbPath = ConfigurationManager.getProperty("usage-statistics", "dbfile");
try {
File dbFile = new File(dbPath);
geoipLookup = new DatabaseReader.Builder(dbFile).build();
} catch (FileNotFoundException fe) {
log.error(
"The GeoLite Database file is missing (" + dbPath + ")! Solr Statistics cannot generate location " +
"based reports! Please see the DSpace installation instructions for instructions to install this " +
"file.",
fe);
} catch (IOException e) {
log.error(
"Unable to load GeoLite Database file (" + dbPath + ")! You may need to reinstall it. See the DSpace " +
"installation instructions for more details.",
e);
}