Apply patches from issue

This commit is contained in:
Mark H. Wood
2012-07-03 16:12:07 -04:00
parent f05ceff8e8
commit b5eb587053
3 changed files with 72 additions and 59 deletions

View File

@@ -45,29 +45,39 @@ public class ApacheLogRobotsProcessor {
CommandLine line = parser.parse(options, args); CommandLine line = parser.parse(options, args);
// Log source
String logFileLoc; String logFileLoc;
String spiderIpPath;
if (line.hasOption("l")) if (line.hasOption("l"))
{ {
logFileLoc = line.getOptionValue("l"); logFileLoc = line.getOptionValue("l");
} }
else { else {
System.out.println("We need our log file"); logFileLoc = "-";
return;
} }
// Spider IP list
String spiderIpPath;
if (line.hasOption("s")) if (line.hasOption("s"))
{ {
spiderIpPath = line.getOptionValue("s"); spiderIpPath = line.getOptionValue("s");
} }
else { else {
System.out.println("We need a spider IP output file"); spiderIpPath = "-";
return;
} }
File spiderIpFile = new File(spiderIpPath);
//Get the IPs already added in our file //Get the IPs already added in our file
Set<String> logSpiders; Set<String> logSpiders;
Writer output;
if ("-".equals(spiderIpPath))
{
logSpiders = new HashSet<String>();
output = new BufferedWriter(new OutputStreamWriter(System.out));
}
else
{
File spiderIpFile = new File(spiderIpPath);
if (spiderIpFile.exists()) if (spiderIpFile.exists())
{ {
logSpiders = SpiderDetector.readIpAddresses(spiderIpFile); logSpiders = SpiderDetector.readIpAddresses(spiderIpFile);
@@ -76,9 +86,16 @@ public class ApacheLogRobotsProcessor {
{ {
logSpiders = new HashSet<String>(); logSpiders = new HashSet<String>();
} }
output = new BufferedWriter(new FileWriter(spiderIpFile));
}
//First read in our log file line per line //First read in our log file line per line
BufferedReader in = new BufferedReader(new FileReader(logFileLoc)); BufferedReader in;
if ("-".equals(logFileLoc))
in = new BufferedReader(new InputStreamReader(System.in));
else
in = new BufferedReader(new FileReader(logFileLoc));
String logLine; String logLine;
while ((logLine = in.readLine()) != null) { while ((logLine = in.readLine()) != null) {
//Currently only check if robot.txt is present in our line //Currently only check if robot.txt is present in our line
@@ -92,11 +109,8 @@ public class ApacheLogRobotsProcessor {
in.close(); in.close();
//Last but not least add the IPs to our file //Last but not least add the IPs to our file
BufferedWriter output = new BufferedWriter(new FileWriter(spiderIpFile));
//Second write the new IPs
for (String ip : logSpiders) { for (String ip : logSpiders) {
System.out.println("Adding new ip: " + ip); System.err.println("Adding new ip: " + ip);
//Write each new IP on a separate line //Write each new IP on a separate line
output.write(ip + "\n"); output.write(ip + "\n");
} }

View File

@@ -85,16 +85,37 @@ public class ClassicDSpaceLogConverter {
int counter = 0; int counter = 0;
int lines = 0; int lines = 0;
// Figure out input, output
BufferedReader input;
Writer output;
try {
if (null == in || in.isEmpty() || "-".equals(in))
{
input = new BufferedReader(new InputStreamReader(System.in));
in = "standard input";
}
else
input = new BufferedReader(new FileReader(in));
if (null == out || out.isEmpty() || "-".equals(out))
{
output = new BufferedWriter(new OutputStreamWriter(System.out));
out = "standard output";
}
else
output = new BufferedWriter(new FileWriter(out));
} catch (IOException ie) {
log.error("File access problem", ie);
return 0;
}
// Say what we're going to do // Say what we're going to do
System.out.println(" About to convert '" + in + "' to '" + out + "'"); System.err.println(" About to convert '" + in + "' to '" + out + "'");
// Setup the regular expressions for the log file // Setup the regular expressions for the log file
LogAnalyser.setRegex(in); LogAnalyser.setRegex(in);
// Open the file and read it line by line // Open the file and read it line by line
BufferedReader input = null;
Writer output = null;
try { try {
String line; String line;
LogLine lline; LogLine lline;
@@ -107,9 +128,6 @@ public class ClassicDSpaceLogConverter {
String uid; String uid;
String lastLine = ""; String lastLine = "";
input = new BufferedReader(new FileReader(new File(in)));
output = new BufferedWriter(new FileWriter(new File(out)));
while ((line = input.readLine()) != null) while ((line = input.readLine()) != null)
{ {
// Read inthe line and covnert it to a LogLine // Read inthe line and covnert it to a LogLine
@@ -236,7 +254,7 @@ public class ClassicDSpaceLogConverter {
} }
// Tell the user what we have done // Tell the user what we have done
System.out.println(" Read " + lines + " lines and recorded " + counter + " events"); System.err.println(" Read " + lines + " lines and recorded " + counter + " events");
return counter; return counter;
} }
@@ -251,7 +269,7 @@ public class ClassicDSpaceLogConverter {
// print the help message // print the help message
HelpFormatter myhelp = new HelpFormatter(); HelpFormatter myhelp = new HelpFormatter();
myhelp.printHelp("ClassicDSpaceLogConverter\n", options); myhelp.printHelp("ClassicDSpaceLogConverter\n", options);
System.out.println("\n\tClassicDSpaceLogConverter -i infilename -o outfilename -v (for verbose output)"); System.err.println("\n\tClassicDSpaceLogConverter -i infilename -o outfilename -v (for verbose output)");
System.exit(exitCode); System.exit(exitCode);
} }
@@ -266,8 +284,8 @@ public class ClassicDSpaceLogConverter {
Options options = new Options(); Options options = new Options();
options.addOption("i", "in", true, "source file"); options.addOption("i", "in", true, "source file ('-' or omit for standard input)");
options.addOption("o", "out", true, "destination directory"); options.addOption("o", "out", true, "destination file or directory ('-' or omit for standard output)");
options.addOption("m", "multiple",false, "treat the input file as having a wildcard ending"); options.addOption("m", "multiple",false, "treat the input file as having a wildcard ending");
options.addOption("n", "newformat",false, "process new format log lines (1.6+)"); options.addOption("n", "newformat",false, "process new format log lines (1.6+)");
options.addOption("v", "verbose", false, "display verbose output (useful for debugging)"); options.addOption("v", "verbose", false, "display verbose output (useful for debugging)");
@@ -292,24 +310,6 @@ public class ClassicDSpaceLogConverter {
printHelp(options, 0); printHelp(options, 0);
} }
// Check we have an input and output file
if ((!line.hasOption('i')) && (!line.hasOption('o')))
{
System.err.println("-i and -o input and output file names are required");
printHelp(options, 1);
}
else if (!line.hasOption('i'))
{
System.err.println("-i input file name is required");
printHelp(options, 1);
}
if (!line.hasOption('o'))
{
System.err.println("-o output file names is required");
printHelp(options, 1);
}
// Whether or not to include event created by org.dspace.usage.LoggerUsageEventListener // Whether or not to include event created by org.dspace.usage.LoggerUsageEventListener
boolean newEvents = line.hasOption('n'); boolean newEvents = line.hasOption('n');
@@ -357,7 +357,7 @@ public class ClassicDSpaceLogConverter {
String[] children = dir.list(filter); String[] children = dir.list(filter);
for (String in : children) for (String in : children)
{ {
System.out.println(in); System.err.println(in);
String out = line.getOptionValue('o') + String out = line.getOptionValue('o') +
(dir.getAbsolutePath() + (dir.getAbsolutePath() +
System.getProperty("file.separator") + in).substring(line.getOptionValue('i').length()); System.getProperty("file.separator") + in).substring(line.getOptionValue('i').length());

View File

@@ -13,7 +13,6 @@ import org.apache.log4j.Logger;
import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer; import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.SolrServerException;
import org.dspace.authorize.AuthorizeException;
import org.dspace.content.*; import org.dspace.content.*;
import org.dspace.content.Collection; import org.dspace.content.Collection;
import org.dspace.core.Context; import org.dspace.core.Context;
@@ -22,7 +21,6 @@ import org.dspace.core.ConfigurationManager;
import org.dspace.eperson.EPerson; import org.dspace.eperson.EPerson;
import org.dspace.statistics.SolrLogger; import org.dspace.statistics.SolrLogger;
import java.sql.SQLException;
import java.text.*; import java.text.*;
import java.io.*; import java.io.*;
import java.util.*; import java.util.*;
@@ -135,7 +133,7 @@ public class StatisticsImporter
} }
/** /**
* Method to load the lines from the statics file and load them into solr * Read lines from the statistics file and load their data into solr.
* *
* @param filename The filename of the file to load * @param filename The filename of the file to load
* @param context The DSpace Context * @param context The DSpace Context
@@ -143,9 +141,6 @@ public class StatisticsImporter
*/ */
private void load(String filename, Context context, boolean verbose) private void load(String filename, Context context, boolean verbose)
{ {
// Print out the filename for confirmation
System.out.println("Processing file: " + filename);
// Item counter // Item counter
int counter = 0; int counter = 0;
int errors = 0; int errors = 0;
@@ -153,7 +148,17 @@ public class StatisticsImporter
try try
{ {
BufferedReader input = new BufferedReader(new FileReader(new File(filename))); BufferedReader input;
if (null == filename || "-".equals(filename))
{
input = new BufferedReader(new InputStreamReader(System.in));
filename = "standard input";
}
else
input = new BufferedReader(new FileReader(new File(filename)));
// Print out the filename for confirmation
System.out.println("Processing file: " + filename);
String line; String line;
// String uuid; // String uuid;
@@ -426,7 +431,7 @@ public class StatisticsImporter
Options options = new Options(); Options options = new Options();
options.addOption("i", "in", true, "the inpout file"); options.addOption("i", "in", true, "the input file ('-' or omit for standard input)");
options.addOption("l", "local", false, "developers tool - map external log file to local handles"); options.addOption("l", "local", false, "developers tool - map external log file to local handles");
options.addOption("m", "multiple", false, "treat the input file as having a wildcard ending"); options.addOption("m", "multiple", false, "treat the input file as having a wildcard ending");
options.addOption("s", "skipdns", false, "skip performing reverse DNS lookups on IP addresses"); options.addOption("s", "skipdns", false, "skip performing reverse DNS lookups on IP addresses");
@@ -441,12 +446,6 @@ public class StatisticsImporter
printHelp(options, 0); printHelp(options, 0);
} }
if (!line.hasOption('i'))
{
System.err.println("You must specify an input file using the -i flag");
printHelp(options, 1);
}
if (line.hasOption('s')) if (line.hasOption('s'))
{ {
skipReverseDNS = true; skipReverseDNS = true;