hazza/DSpace
1
0
Fork 0
mirror of https://github.com/DSpace/DSpace.git synced 2025-10-19 16:03:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

[DS-3832] Upgrade to GeoIP2.

Browse Source
This commit is contained in:
Mark H. Wood
2018-03-21 15:49:38 -04:00
parent c3d567c31e
commit 074337f167
7 changed files with 194 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
dspace-api/pom.xml
View File

@@ -570,9 +570,9 @@
<artifactId>commons-configuration</artifactId>
</dependency>
<dependency>
<groupId>com.maxmind.geoip</groupId>
<artifactId>geoip-api</artifactId>
<version>1.3.0</version>
<groupId>com.maxmind.geoip2</groupId>
<artifactId>geoip2</artifactId>
<version>2.11.0</version>
</dependency>
<dependency>
<groupId>org.apache.ant</groupId>

102
dspace-api/src/main/java/org/dspace/statistics/SolrLoggerServiceImpl.java
View File

@@ -16,6 +16,7 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.net.InetAddress;
import java.net.URLEncoder;
import java.sql.SQLException;
import java.text.DateFormat;
@@ -34,8 +35,9 @@ import javax.servlet.http.HttpServletRequest;
import au.com.bytecode.opencsv.CSVReader;
import au.com.bytecode.opencsv.CSVWriter;
import com.maxmind.geoip.Location;
import com.maxmind.geoip.LookupService;
import com.maxmind.geoip2.DatabaseReader;
import com.maxmind.geoip2.exception.GeoIp2Exception;
import com.maxmind.geoip2.model.CityResponse;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
@@ -108,7 +110,7 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea
public static final String DATE_FORMAT_DCDATE = "yyyy-MM-dd'T'HH:mm:ss'Z'";
protected LookupService locationService;
protected DatabaseReader locationService;
protected boolean useProxies;
@@ -164,22 +166,22 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea
// Read in the file so we don't have to do it all the time
//spiderIps = SpiderDetector.getSpiderIpAddresses();
LookupService service = null;
DatabaseReader service = null;
// Get the db file for the location
String dbfile = configurationService.getProperty("usage-statistics.dbfile");
if (dbfile != null) {
String dbPath = configurationService.getProperty("usage-statistics.dbfile");
if (dbPath != null) {
try {
service = new LookupService(dbfile,
LookupService.GEOIP_STANDARD);
File dbFile = new File(dbPath);
service = new DatabaseReader.Builder(dbFile).build();
} catch (FileNotFoundException fe) {
log.error(
"The GeoLite Database file is missing (" + dbfile + ")! Solr Statistics cannot generate location " +
"The GeoLite Database file is missing (" + dbPath + ")! Solr Statistics cannot generate location " +
"based reports! Please see the DSpace installation instructions for instructions to install " +
"this file.",
fe);
} catch (IOException e) {
log.error(
"Unable to load GeoLite Database file (" + dbfile + ")! You may need to reinstall it. See the " +
"Unable to load GeoLite Database file (" + dbPath + ")! You may need to reinstall it. See the " +
"DSpace installation instructions for more details.",
e);
}
@@ -332,23 +334,31 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea
// Save the location information if valid, save the event without
// location information if not valid
if (locationService != null) {
Location location = locationService.getLocation(ip);
if (location != null
&& !("--".equals(location.countryCode)
&& location.latitude == -180 && location.longitude == -180)) {
try {
doc1.addField("continent", LocationUtils
.getContinentCode(location.countryCode));
} catch (Exception e) {
System.out
.println("COUNTRY ERROR: " + location.countryCode);
try {
InetAddress ipAddress = InetAddress.getByName(ip);
CityResponse location = locationService.city(ipAddress);
String countryCode = location.getCountry().getIsoCode();
double latitude = location.getLocation().getLatitude();
double longitude = location.getLocation().getLongitude();
if (!(
"--".equals(countryCode)
&& latitude == -180
&& longitude == -180)
) {
try {
doc1.addField("continent", LocationUtils
.getContinentCode(countryCode));
} catch (Exception e) {
System.out
.println("COUNTRY ERROR: " + countryCode);
}
doc1.addField("countryCode", countryCode);
doc1.addField("city", location.getCity().getName());
doc1.addField("latitude", latitude);
doc1.addField("longitude", longitude);
}
doc1.addField("countryCode", location.countryCode);
doc1.addField("city", location.city);
doc1.addField("latitude", location.latitude);
doc1.addField("longitude", location.longitude);
} catch (IOException | GeoIp2Exception e) {
log.error("Unable to get location of request", e);
}
}
}
@@ -411,23 +421,31 @@ public class SolrLoggerServiceImpl implements SolrLoggerService, InitializingBea
// Save the location information if valid, save the event without
// location information if not valid
if (locationService != null) {
Location location = locationService.getLocation(ip);
if (location != null
&& !("--".equals(location.countryCode)
&& location.latitude == -180 && location.longitude == -180)) {
try {
doc1.addField("continent", LocationUtils
.getContinentCode(location.countryCode));
} catch (Exception e) {
System.out
.println("COUNTRY ERROR: " + location.countryCode);
try {
InetAddress ipAddress = InetAddress.getByName(ip);
CityResponse location = locationService.city(ipAddress);
String countryCode = location.getCountry().getIsoCode();
double latitude = location.getLocation().getLatitude();
double longitude = location.getLocation().getLongitude();
if (!(
"--".equals(countryCode)
&& latitude == -180
&& longitude == -180)
) {
try {
doc1.addField("continent", LocationUtils
.getContinentCode(countryCode));
} catch (Exception e) {
System.out
.println("COUNTRY ERROR: " + countryCode);
}
doc1.addField("countryCode", countryCode);
doc1.addField("city", location.getCity().getName());
doc1.addField("latitude", latitude);
doc1.addField("longitude", longitude);
}
doc1.addField("countryCode", location.countryCode);
doc1.addField("city", location.city);
doc1.addField("latitude", location.latitude);
doc1.addField("longitude", location.longitude);
} catch (GeoIp2Exception | IOException e) {
log.error("Unable to get location of request", e);
}
}
}

34
dspace-api/src/main/java/org/dspace/statistics/util/StatisticsImporter.java
View File

@@ -14,6 +14,7 @@ import java.io.FileReader;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.InetAddress;
import java.text.DateFormat;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
@@ -25,8 +26,8 @@ import java.util.List;
import java.util.Random;
import java.util.UUID;
import com.maxmind.geoip.Location;
import com.maxmind.geoip.LookupService;
import com.maxmind.geoip2.DatabaseReader;
import com.maxmind.geoip2.model.CityResponse;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
@@ -84,7 +85,7 @@ public class StatisticsImporter {
/**
* GEOIP lookup service
*/
private static LookupService geoipLookup;
private static DatabaseReader geoipLookup;
/**
* Whether to skip the DNS reverse lookup or not
@@ -219,8 +220,8 @@ public class StatisticsImporter {
String continent = "";
String country = "";
String countryCode = "";
float longitude = 0f;
float latitude = 0f;
double longitude = 0f;
double latitude = 0f;
String city = "";
String dns;
@@ -275,14 +276,14 @@ public class StatisticsImporter {
}
// Get the geo information for the user
Location location;
try {
location = geoipLookup.getLocation(ip);
city = location.city;
country = location.countryName;
countryCode = location.countryCode;
longitude = location.longitude;
latitude = location.latitude;
InetAddress ipAddress = InetAddress.getByName(ip);
CityResponse cityResponse = geoipLookup.city(ipAddress);
city = cityResponse.getCity().getName();
country = cityResponse.getCountry().getName();
countryCode = cityResponse.getCountry().getIsoCode();
longitude = cityResponse.getLocation().getLongitude();
latitude = cityResponse.getLocation().getLatitude();
if (verbose) {
data += (", country = " + country);
data += (", city = " + city);
@@ -469,18 +470,19 @@ public class StatisticsImporter {
}
solr = new HttpSolrServer(sserver);
String dbfile = ConfigurationManager.getProperty("usage-statistics", "dbfile");
String dbPath = ConfigurationManager.getProperty("usage-statistics", "dbfile");
try {
geoipLookup = new LookupService(dbfile, LookupService.GEOIP_STANDARD);
File dbFile = new File(dbPath);
geoipLookup = new DatabaseReader.Builder(dbFile).build();
} catch (FileNotFoundException fe) {
log.error(
"The GeoLite Database file is missing (" + dbfile + ")! Solr Statistics cannot generate location " +
"The GeoLite Database file is missing (" + dbPath + ")! Solr Statistics cannot generate location " +
"based reports! Please see the DSpace installation instructions for instructions to install this " +
"file.",
fe);
} catch (IOException e) {
log.error(
"Unable to load GeoLite Database file (" + dbfile + ")! You may need to reinstall it. See the DSpace " +
"Unable to load GeoLite Database file (" + dbPath + ")! You may need to reinstall it. See the DSpace " +
"installation instructions for more details.",
e);
}

64
dspace-api/src/test/java/org/dspace/statistics/MockLookupService.java
View File

@@ -7,23 +7,31 @@
*/
package org.dspace.statistics;
import java.io.File;
import java.io.IOException;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import com.maxmind.geoip.Location;
import com.maxmind.geoip.LookupService;
import org.dspace.AbstractDSpaceTest;
import com.maxmind.geoip2.DatabaseReader;
import com.maxmind.geoip2.model.CityResponse;
import com.maxmind.geoip2.record.City;
import com.maxmind.geoip2.record.Continent;
import com.maxmind.geoip2.record.Country;
import com.maxmind.geoip2.record.Location;
import com.maxmind.geoip2.record.MaxMind;
import com.maxmind.geoip2.record.Postal;
import com.maxmind.geoip2.record.RepresentedCountry;
import com.maxmind.geoip2.record.Subdivision;
import com.maxmind.geoip2.record.Traits;
import mockit.Mock;
import mockit.MockUp;
/**
* Mock service to mock the location Lookup Service used by the SOLR statistics logger
*/
public class MockLookupService extends LookupService {
public MockLookupService() throws IOException {
//Just give our super class a file so that he's happy. We'll mock all responses anyway.
super(AbstractDSpaceTest.getDspaceDir() + File.separator + "config" + File.separator + "dspace.cfg");
}
public class MockLookupService
extends MockUp<DatabaseReader> {
/*
@Override
public Location getLocation(String str) {
Location location = new Location();
@@ -40,4 +48,36 @@ public class MockLookupService extends LookupService {
return location;
}
*/
@Mock
public CityResponse city(InetAddress address) {
List<String> names = new ArrayList<>(1);
names.add("New York");
City city = new City(names, 1, 1, new HashMap());
Continent continent = new Continent();
names.clear();
names.add("United States");
Country country = new Country(names, 1, 1, "US", new HashMap());
Location location = new Location(1, 1, 40.760498D, -73.9933D, 501, 1, "EST");
MaxMind maxmind = new MaxMind();
Postal postal = new Postal("10036", 1);
RepresentedCountry representedCountry = new RepresentedCountry();
ArrayList<Subdivision> subdivisions = new ArrayList<>(0);
Traits traits = new Traits();
CityResponse response = new CityResponse(city, continent, country,
location, maxmind, postal, country, representedCountry,
subdivisions, traits);
return response;
}
}

7
dspace-api/src/test/java/org/dspace/statistics/MockSolrLoggerServiceImpl.java
View File

@@ -7,6 +7,9 @@
*/
package org.dspace.statistics;
import java.io.File;
import com.maxmind.geoip2.DatabaseReader;
import org.dspace.services.ConfigurationService;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
@@ -21,9 +24,9 @@ public class MockSolrLoggerServiceImpl extends SolrLoggerServiceImpl implements
@Override
public void afterPropertiesSet() throws Exception {
//We don' use SOLR in the tests of this module
//We don't use SOLR in the tests of this module
solr = null;
locationService = new MockLookupService();
locationService = new DatabaseReader.Builder(new File(".")).build();
useProxies = configurationService.getBooleanProperty("useProxies");
}

58
dspace-spring-rest/src/test/java/org/dspace/statistics/MockLookupService.java
View File

@@ -7,23 +7,35 @@
*/
package org.dspace.statistics;
import java.io.File;
import java.io.IOException;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import com.maxmind.geoip.Location;
import com.maxmind.geoip.LookupService;
import org.dspace.app.rest.test.AbstractDSpaceIntegrationTest;
import com.maxmind.geoip2.DatabaseReader;
import com.maxmind.geoip2.model.CityResponse;
import com.maxmind.geoip2.record.City;
import com.maxmind.geoip2.record.Continent;
import com.maxmind.geoip2.record.Country;
import com.maxmind.geoip2.record.Location;
import com.maxmind.geoip2.record.MaxMind;
import com.maxmind.geoip2.record.Postal;
import com.maxmind.geoip2.record.RepresentedCountry;
import com.maxmind.geoip2.record.Subdivision;
import com.maxmind.geoip2.record.Traits;
import mockit.Mock;
import mockit.MockUp;
/**
* Mock service to mock the location Lookup Service used by the SOLR statistics logger
*/
public class MockLookupService extends LookupService {
public class MockLookupService extends MockUp<DatabaseReader> {
public MockLookupService() throws IOException {
//Just give our super class a file so that he's happy. We'll mock all responses anyway.
super(AbstractDSpaceIntegrationTest.getDspaceDir() + File.separator + "config" + File.separator + "dspace.cfg");
}
/*
@Override
public Location getLocation(String str) {
Location location = new Location();
@@ -40,4 +52,36 @@ public class MockLookupService extends LookupService {
return location;
}
*/
@Mock
public CityResponse city(InetAddress address) {
List<String> names = new ArrayList<>(1);
names.add("New York");
City city = new City(names, 1, 1, new HashMap());
Continent continent = new Continent();
names.clear();
names.add("United States");
Country country = new Country(names, 1, 1, "US", new HashMap());
Location location = new Location(1, 1, 40.760498D, -73.9933D, 501, 1, "EST");
MaxMind maxmind = new MaxMind();
Postal postal = new Postal("10036", 1);
RepresentedCountry representedCountry = new RepresentedCountry();
ArrayList<Subdivision> subdivisions = new ArrayList<>(0);
Traits traits = new Traits();
CityResponse response = new CityResponse(city, continent, country,
location, maxmind, postal, country, representedCountry,
subdivisions, traits);
return response;
}
}

6
dspace-spring-rest/src/test/java/org/dspace/statistics/MockSolrLoggerServiceImpl.java
View File

@@ -7,6 +7,9 @@
*/
package org.dspace.statistics;
import java.io.File;
import com.maxmind.geoip2.DatabaseReader;
import org.dspace.services.ConfigurationService;
import org.dspace.solr.MockSolrServer;
import org.springframework.beans.factory.DisposableBean;
@@ -27,10 +30,11 @@ public class MockSolrLoggerServiceImpl extends SolrLoggerServiceImpl implements
public void afterPropertiesSet() throws Exception {
mockSolrServer = new MockSolrServer("statistics");
solr = mockSolrServer.getSolrServer();
locationService = new MockLookupService();
locationService = new DatabaseReader.Builder(new File(".")).build();
useProxies = configurationService.getBooleanProperty("useProxies");
}
@Override
public void destroy() throws Exception {
mockSolrServer.destroy();
}