mirror of
https://github.com/DSpace/DSpace.git
synced 2025-10-08 02:24:18 +00:00
Apply patches from issue
This commit is contained in:
@@ -45,29 +45,39 @@ public class ApacheLogRobotsProcessor {
|
|||||||
|
|
||||||
CommandLine line = parser.parse(options, args);
|
CommandLine line = parser.parse(options, args);
|
||||||
|
|
||||||
|
// Log source
|
||||||
String logFileLoc;
|
String logFileLoc;
|
||||||
String spiderIpPath;
|
|
||||||
if (line.hasOption("l"))
|
if (line.hasOption("l"))
|
||||||
{
|
{
|
||||||
logFileLoc = line.getOptionValue("l");
|
logFileLoc = line.getOptionValue("l");
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
System.out.println("We need our log file");
|
logFileLoc = "-";
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Spider IP list
|
||||||
|
String spiderIpPath;
|
||||||
if (line.hasOption("s"))
|
if (line.hasOption("s"))
|
||||||
{
|
{
|
||||||
spiderIpPath = line.getOptionValue("s");
|
spiderIpPath = line.getOptionValue("s");
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
System.out.println("We need a spider IP output file");
|
spiderIpPath = "-";
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
File spiderIpFile = new File(spiderIpPath);
|
|
||||||
|
|
||||||
//Get the IPs already added in our file
|
//Get the IPs already added in our file
|
||||||
Set<String> logSpiders;
|
Set<String> logSpiders;
|
||||||
|
Writer output;
|
||||||
|
|
||||||
|
if ("-".equals(spiderIpPath))
|
||||||
|
{
|
||||||
|
logSpiders = new HashSet<String>();
|
||||||
|
output = new BufferedWriter(new OutputStreamWriter(System.out));
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
File spiderIpFile = new File(spiderIpPath);
|
||||||
|
|
||||||
if (spiderIpFile.exists())
|
if (spiderIpFile.exists())
|
||||||
{
|
{
|
||||||
logSpiders = SpiderDetector.readIpAddresses(spiderIpFile);
|
logSpiders = SpiderDetector.readIpAddresses(spiderIpFile);
|
||||||
@@ -76,9 +86,16 @@ public class ApacheLogRobotsProcessor {
|
|||||||
{
|
{
|
||||||
logSpiders = new HashSet<String>();
|
logSpiders = new HashSet<String>();
|
||||||
}
|
}
|
||||||
|
output = new BufferedWriter(new FileWriter(spiderIpFile));
|
||||||
|
}
|
||||||
|
|
||||||
//First read in our log file line per line
|
//First read in our log file line per line
|
||||||
BufferedReader in = new BufferedReader(new FileReader(logFileLoc));
|
BufferedReader in;
|
||||||
|
if ("-".equals(logFileLoc))
|
||||||
|
in = new BufferedReader(new InputStreamReader(System.in));
|
||||||
|
else
|
||||||
|
in = new BufferedReader(new FileReader(logFileLoc));
|
||||||
|
|
||||||
String logLine;
|
String logLine;
|
||||||
while ((logLine = in.readLine()) != null) {
|
while ((logLine = in.readLine()) != null) {
|
||||||
//Currently only check if robot.txt is present in our line
|
//Currently only check if robot.txt is present in our line
|
||||||
@@ -92,11 +109,8 @@ public class ApacheLogRobotsProcessor {
|
|||||||
in.close();
|
in.close();
|
||||||
|
|
||||||
//Last but not least add the IPs to our file
|
//Last but not least add the IPs to our file
|
||||||
BufferedWriter output = new BufferedWriter(new FileWriter(spiderIpFile));
|
|
||||||
|
|
||||||
//Second write the new IPs
|
|
||||||
for (String ip : logSpiders) {
|
for (String ip : logSpiders) {
|
||||||
System.out.println("Adding new ip: " + ip);
|
System.err.println("Adding new ip: " + ip);
|
||||||
//Write each new IP on a separate line
|
//Write each new IP on a separate line
|
||||||
output.write(ip + "\n");
|
output.write(ip + "\n");
|
||||||
}
|
}
|
||||||
|
@@ -85,16 +85,37 @@ public class ClassicDSpaceLogConverter {
|
|||||||
int counter = 0;
|
int counter = 0;
|
||||||
int lines = 0;
|
int lines = 0;
|
||||||
|
|
||||||
|
// Figure out input, output
|
||||||
|
BufferedReader input;
|
||||||
|
Writer output;
|
||||||
|
try {
|
||||||
|
if (null == in || in.isEmpty() || "-".equals(in))
|
||||||
|
{
|
||||||
|
input = new BufferedReader(new InputStreamReader(System.in));
|
||||||
|
in = "standard input";
|
||||||
|
}
|
||||||
|
else
|
||||||
|
input = new BufferedReader(new FileReader(in));
|
||||||
|
|
||||||
|
if (null == out || out.isEmpty() || "-".equals(out))
|
||||||
|
{
|
||||||
|
output = new BufferedWriter(new OutputStreamWriter(System.out));
|
||||||
|
out = "standard output";
|
||||||
|
}
|
||||||
|
else
|
||||||
|
output = new BufferedWriter(new FileWriter(out));
|
||||||
|
} catch (IOException ie) {
|
||||||
|
log.error("File access problem", ie);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
// Say what we're going to do
|
// Say what we're going to do
|
||||||
System.out.println(" About to convert '" + in + "' to '" + out + "'");
|
System.err.println(" About to convert '" + in + "' to '" + out + "'");
|
||||||
|
|
||||||
// Setup the regular expressions for the log file
|
// Setup the regular expressions for the log file
|
||||||
LogAnalyser.setRegex(in);
|
LogAnalyser.setRegex(in);
|
||||||
|
|
||||||
// Open the file and read it line by line
|
// Open the file and read it line by line
|
||||||
BufferedReader input = null;
|
|
||||||
Writer output = null;
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
String line;
|
String line;
|
||||||
LogLine lline;
|
LogLine lline;
|
||||||
@@ -107,9 +128,6 @@ public class ClassicDSpaceLogConverter {
|
|||||||
String uid;
|
String uid;
|
||||||
String lastLine = "";
|
String lastLine = "";
|
||||||
|
|
||||||
input = new BufferedReader(new FileReader(new File(in)));
|
|
||||||
output = new BufferedWriter(new FileWriter(new File(out)));
|
|
||||||
|
|
||||||
while ((line = input.readLine()) != null)
|
while ((line = input.readLine()) != null)
|
||||||
{
|
{
|
||||||
// Read inthe line and covnert it to a LogLine
|
// Read inthe line and covnert it to a LogLine
|
||||||
@@ -236,7 +254,7 @@ public class ClassicDSpaceLogConverter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Tell the user what we have done
|
// Tell the user what we have done
|
||||||
System.out.println(" Read " + lines + " lines and recorded " + counter + " events");
|
System.err.println(" Read " + lines + " lines and recorded " + counter + " events");
|
||||||
return counter;
|
return counter;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -251,7 +269,7 @@ public class ClassicDSpaceLogConverter {
|
|||||||
// print the help message
|
// print the help message
|
||||||
HelpFormatter myhelp = new HelpFormatter();
|
HelpFormatter myhelp = new HelpFormatter();
|
||||||
myhelp.printHelp("ClassicDSpaceLogConverter\n", options);
|
myhelp.printHelp("ClassicDSpaceLogConverter\n", options);
|
||||||
System.out.println("\n\tClassicDSpaceLogConverter -i infilename -o outfilename -v (for verbose output)");
|
System.err.println("\n\tClassicDSpaceLogConverter -i infilename -o outfilename -v (for verbose output)");
|
||||||
System.exit(exitCode);
|
System.exit(exitCode);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -266,8 +284,8 @@ public class ClassicDSpaceLogConverter {
|
|||||||
|
|
||||||
Options options = new Options();
|
Options options = new Options();
|
||||||
|
|
||||||
options.addOption("i", "in", true, "source file");
|
options.addOption("i", "in", true, "source file ('-' or omit for standard input)");
|
||||||
options.addOption("o", "out", true, "destination directory");
|
options.addOption("o", "out", true, "destination file or directory ('-' or omit for standard output)");
|
||||||
options.addOption("m", "multiple",false, "treat the input file as having a wildcard ending");
|
options.addOption("m", "multiple",false, "treat the input file as having a wildcard ending");
|
||||||
options.addOption("n", "newformat",false, "process new format log lines (1.6+)");
|
options.addOption("n", "newformat",false, "process new format log lines (1.6+)");
|
||||||
options.addOption("v", "verbose", false, "display verbose output (useful for debugging)");
|
options.addOption("v", "verbose", false, "display verbose output (useful for debugging)");
|
||||||
@@ -292,24 +310,6 @@ public class ClassicDSpaceLogConverter {
|
|||||||
printHelp(options, 0);
|
printHelp(options, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check we have an input and output file
|
|
||||||
if ((!line.hasOption('i')) && (!line.hasOption('o')))
|
|
||||||
{
|
|
||||||
System.err.println("-i and -o input and output file names are required");
|
|
||||||
printHelp(options, 1);
|
|
||||||
}
|
|
||||||
else if (!line.hasOption('i'))
|
|
||||||
{
|
|
||||||
System.err.println("-i input file name is required");
|
|
||||||
printHelp(options, 1);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!line.hasOption('o'))
|
|
||||||
{
|
|
||||||
System.err.println("-o output file names is required");
|
|
||||||
printHelp(options, 1);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Whether or not to include event created by org.dspace.usage.LoggerUsageEventListener
|
// Whether or not to include event created by org.dspace.usage.LoggerUsageEventListener
|
||||||
boolean newEvents = line.hasOption('n');
|
boolean newEvents = line.hasOption('n');
|
||||||
|
|
||||||
@@ -357,7 +357,7 @@ public class ClassicDSpaceLogConverter {
|
|||||||
String[] children = dir.list(filter);
|
String[] children = dir.list(filter);
|
||||||
for (String in : children)
|
for (String in : children)
|
||||||
{
|
{
|
||||||
System.out.println(in);
|
System.err.println(in);
|
||||||
String out = line.getOptionValue('o') +
|
String out = line.getOptionValue('o') +
|
||||||
(dir.getAbsolutePath() +
|
(dir.getAbsolutePath() +
|
||||||
System.getProperty("file.separator") + in).substring(line.getOptionValue('i').length());
|
System.getProperty("file.separator") + in).substring(line.getOptionValue('i').length());
|
||||||
|
@@ -13,7 +13,6 @@ import org.apache.log4j.Logger;
|
|||||||
import org.apache.solr.common.SolrInputDocument;
|
import org.apache.solr.common.SolrInputDocument;
|
||||||
import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
|
import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
|
||||||
import org.apache.solr.client.solrj.SolrServerException;
|
import org.apache.solr.client.solrj.SolrServerException;
|
||||||
import org.dspace.authorize.AuthorizeException;
|
|
||||||
import org.dspace.content.*;
|
import org.dspace.content.*;
|
||||||
import org.dspace.content.Collection;
|
import org.dspace.content.Collection;
|
||||||
import org.dspace.core.Context;
|
import org.dspace.core.Context;
|
||||||
@@ -22,7 +21,6 @@ import org.dspace.core.ConfigurationManager;
|
|||||||
import org.dspace.eperson.EPerson;
|
import org.dspace.eperson.EPerson;
|
||||||
import org.dspace.statistics.SolrLogger;
|
import org.dspace.statistics.SolrLogger;
|
||||||
|
|
||||||
import java.sql.SQLException;
|
|
||||||
import java.text.*;
|
import java.text.*;
|
||||||
import java.io.*;
|
import java.io.*;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
@@ -135,7 +133,7 @@ public class StatisticsImporter
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Method to load the lines from the statics file and load them into solr
|
* Read lines from the statistics file and load their data into solr.
|
||||||
*
|
*
|
||||||
* @param filename The filename of the file to load
|
* @param filename The filename of the file to load
|
||||||
* @param context The DSpace Context
|
* @param context The DSpace Context
|
||||||
@@ -143,9 +141,6 @@ public class StatisticsImporter
|
|||||||
*/
|
*/
|
||||||
private void load(String filename, Context context, boolean verbose)
|
private void load(String filename, Context context, boolean verbose)
|
||||||
{
|
{
|
||||||
// Print out the filename for confirmation
|
|
||||||
System.out.println("Processing file: " + filename);
|
|
||||||
|
|
||||||
// Item counter
|
// Item counter
|
||||||
int counter = 0;
|
int counter = 0;
|
||||||
int errors = 0;
|
int errors = 0;
|
||||||
@@ -153,7 +148,17 @@ public class StatisticsImporter
|
|||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
BufferedReader input = new BufferedReader(new FileReader(new File(filename)));
|
BufferedReader input;
|
||||||
|
if (null == filename || "-".equals(filename))
|
||||||
|
{
|
||||||
|
input = new BufferedReader(new InputStreamReader(System.in));
|
||||||
|
filename = "standard input";
|
||||||
|
}
|
||||||
|
else
|
||||||
|
input = new BufferedReader(new FileReader(new File(filename)));
|
||||||
|
|
||||||
|
// Print out the filename for confirmation
|
||||||
|
System.out.println("Processing file: " + filename);
|
||||||
|
|
||||||
String line;
|
String line;
|
||||||
// String uuid;
|
// String uuid;
|
||||||
@@ -426,7 +431,7 @@ public class StatisticsImporter
|
|||||||
|
|
||||||
Options options = new Options();
|
Options options = new Options();
|
||||||
|
|
||||||
options.addOption("i", "in", true, "the inpout file");
|
options.addOption("i", "in", true, "the input file ('-' or omit for standard input)");
|
||||||
options.addOption("l", "local", false, "developers tool - map external log file to local handles");
|
options.addOption("l", "local", false, "developers tool - map external log file to local handles");
|
||||||
options.addOption("m", "multiple", false, "treat the input file as having a wildcard ending");
|
options.addOption("m", "multiple", false, "treat the input file as having a wildcard ending");
|
||||||
options.addOption("s", "skipdns", false, "skip performing reverse DNS lookups on IP addresses");
|
options.addOption("s", "skipdns", false, "skip performing reverse DNS lookups on IP addresses");
|
||||||
@@ -441,12 +446,6 @@ public class StatisticsImporter
|
|||||||
printHelp(options, 0);
|
printHelp(options, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!line.hasOption('i'))
|
|
||||||
{
|
|
||||||
System.err.println("You must specify an input file using the -i flag");
|
|
||||||
printHelp(options, 1);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (line.hasOption('s'))
|
if (line.hasOption('s'))
|
||||||
{
|
{
|
||||||
skipReverseDNS = true;
|
skipReverseDNS = true;
|
||||||
|
Reference in New Issue
Block a user