diff --git a/dspace-api/pom.xml b/dspace-api/pom.xml index 90553cc6e2..24a6f9536a 100644 --- a/dspace-api/pom.xml +++ b/dspace-api/pom.xml @@ -377,7 +377,7 @@ org.hibernate.javax.persistence hibernate-jpa-2.1-api - 1.0.0.Final + 1.0.2.Final @@ -398,7 +398,7 @@ org.ow2.asm asm-commons - + org.bouncycastle bcpkix-jdk15on @@ -534,10 +534,6 @@ org.apache.pdfbox fontbox - - org.apache.poi - poi-scratchpad - xalan xalan @@ -610,27 +606,13 @@ httpmime + org.apache.solr solr-solrj ${solr.client.version} - - - - org.eclipse.jetty - jetty-http - - - org.eclipse.jetty - jetty-io - - - org.eclipse.jetty - jetty-util - - - + org.apache.solr @@ -658,39 +640,10 @@ - - org.apache.solr - solr-cell - - - - org.apache.commons - commons-text - - - - org.eclipse.jetty - jetty-http - - - org.eclipse.jetty - jetty-io - - - org.eclipse.jetty - jetty-util - - - org.apache.lucene lucene-core - - - org.apache.tika - tika-parsers - org.apache.lucene lucene-analyzers-icu @@ -706,9 +659,15 @@ lucene-analyzers-stempel test + + - org.apache.xmlbeans - xmlbeans + org.apache.tika + tika-core + + + org.apache.tika + tika-parsers-standard-package @@ -766,7 +725,7 @@ org.flywaydb flyway-core - 6.5.7 + 8.4.4 @@ -877,6 +836,14 @@ 20180130 + + + com.github.stefanbirkner + system-rules + 1.19.0 + test + + com.opencsv @@ -888,7 +855,6 @@ org.apache.velocity velocity-engine-core - jar @@ -910,13 +876,6 @@ 2.0.0 - - com.github.stefanbirkner - system-rules - 1.19.0 - test - - org.mock-server mockserver-junit-rule @@ -963,7 +922,7 @@ org.apache.velocity velocity-engine-core - 2.2 + 2.3 org.xmlunit diff --git a/dspace-api/src/main/java/org/dspace/app/mediafilter/ExcelFilter.java b/dspace-api/src/main/java/org/dspace/app/mediafilter/ExcelFilter.java deleted file mode 100644 index c17d168c04..0000000000 --- a/dspace-api/src/main/java/org/dspace/app/mediafilter/ExcelFilter.java +++ /dev/null @@ -1,99 +0,0 @@ -/** - * The contents of this file are subject to the license and copyright - * detailed in the LICENSE and NOTICE files at the root of the source - * tree and available online at - * - * http://www.dspace.org/license/ - */ -package org.dspace.app.mediafilter; - -import java.io.InputStream; -import java.nio.charset.StandardCharsets; - -import org.apache.commons.io.IOUtils; -import org.apache.logging.log4j.Logger; -import org.apache.poi.POITextExtractor; -import org.apache.poi.extractor.ExtractorFactory; -import org.apache.poi.hssf.extractor.ExcelExtractor; -import org.apache.poi.xssf.extractor.XSSFExcelExtractor; -import org.dspace.content.Item; - -/* - * ExcelFilter - * - * Entries you must add to dspace.cfg: - * - * filter.plugins = blah, \ - * Excel Text Extractor - * - * plugin.named.org.dspace.app.mediafilter.FormatFilter = \ - * blah = blah, \ - * org.dspace.app.mediafilter.ExcelFilter = Excel Text Extractor - * - * #Configure each filter's input Formats - * filter.org.dspace.app.mediafilter.ExcelFilter.inputFormats = Microsoft Excel, Microsoft Excel XML - * - */ -public class ExcelFilter extends MediaFilter { - - private static Logger log = org.apache.logging.log4j.LogManager.getLogger(ExcelFilter.class); - - public String getFilteredName(String oldFilename) { - return oldFilename + ".txt"; - } - - /** - * @return String bundle name - */ - public String getBundleName() { - return "TEXT"; - } - - /** - * @return String bitstream format - */ - public String getFormatString() { - return "Text"; - } - - /** - * @return String description - */ - public String getDescription() { - return "Extracted text"; - } - - /** - * @param item item - * @param source source input stream - * @param verbose verbose mode - * @return InputStream the resulting input stream - * @throws Exception if error - */ - @Override - public InputStream getDestinationStream(Item item, InputStream source, boolean verbose) - throws Exception { - String extractedText = null; - - try { - POITextExtractor theExtractor = ExtractorFactory.createExtractor(source); - if (theExtractor instanceof ExcelExtractor) { - // for xls file - extractedText = (theExtractor).getText(); - } else if (theExtractor instanceof XSSFExcelExtractor) { - // for xlsx file - extractedText = (theExtractor).getText(); - } - } catch (Exception e) { - log.error("Error filtering bitstream: " + e.getMessage(), e); - throw e; - } - - if (extractedText != null) { - // generate an input stream with the extracted text - return IOUtils.toInputStream(extractedText, StandardCharsets.UTF_8); - } - - return null; - } -} diff --git a/dspace-api/src/main/java/org/dspace/app/mediafilter/HTMLFilter.java b/dspace-api/src/main/java/org/dspace/app/mediafilter/HTMLFilter.java deleted file mode 100644 index 5e10f2841d..0000000000 --- a/dspace-api/src/main/java/org/dspace/app/mediafilter/HTMLFilter.java +++ /dev/null @@ -1,82 +0,0 @@ -/** - * The contents of this file are subject to the license and copyright - * detailed in the LICENSE and NOTICE files at the root of the source - * tree and available online at - * - * http://www.dspace.org/license/ - */ -package org.dspace.app.mediafilter; - -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import javax.swing.text.Document; -import javax.swing.text.html.HTMLEditorKit; - -import org.dspace.content.Item; - -/* - * - * to do: helpful error messages - can't find mediafilter.cfg - can't - * instantiate filter - bitstream format doesn't exist - * - */ -public class HTMLFilter extends MediaFilter { - - @Override - public String getFilteredName(String oldFilename) { - return oldFilename + ".txt"; - } - - /** - * @return String bundle name - */ - @Override - public String getBundleName() { - return "TEXT"; - } - - /** - * @return String bitstream format - */ - @Override - public String getFormatString() { - return "Text"; - } - - /** - * @return String description - */ - @Override - public String getDescription() { - return "Extracted text"; - } - - /** - * @param currentItem item - * @param source source input stream - * @param verbose verbose mode - * @return InputStream the resulting input stream - * @throws Exception if error - */ - @Override - public InputStream getDestinationStream(Item currentItem, InputStream source, boolean verbose) - throws Exception { - // try and read the document - set to ignore character set directive, - // assuming that the input stream is already set properly (I hope) - HTMLEditorKit kit = new HTMLEditorKit(); - Document doc = kit.createDefaultDocument(); - - doc.putProperty("IgnoreCharsetDirective", Boolean.TRUE); - - kit.read(source, doc, 0); - - String extractedText = doc.getText(0, doc.getLength()); - - // generate an input stream with the extracted text - byte[] textBytes = extractedText.getBytes(StandardCharsets.UTF_8); - ByteArrayInputStream bais = new ByteArrayInputStream(textBytes); - - return bais; - } -} diff --git a/dspace-api/src/main/java/org/dspace/app/mediafilter/PDFFilter.java b/dspace-api/src/main/java/org/dspace/app/mediafilter/PDFFilter.java deleted file mode 100644 index c90d7c5a6e..0000000000 --- a/dspace-api/src/main/java/org/dspace/app/mediafilter/PDFFilter.java +++ /dev/null @@ -1,137 +0,0 @@ -/** - * The contents of this file are subject to the license and copyright - * detailed in the LICENSE and NOTICE files at the root of the source - * tree and available online at - * - * http://www.dspace.org/license/ - */ -package org.dspace.app.mediafilter; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.InputStream; -import java.io.OutputStreamWriter; -import java.io.Writer; - -import org.apache.logging.log4j.Logger; -import org.apache.pdfbox.pdmodel.PDDocument; -import org.apache.pdfbox.pdmodel.encryption.InvalidPasswordException; -import org.apache.pdfbox.text.PDFTextStripper; -import org.dspace.content.Item; -import org.dspace.services.ConfigurationService; -import org.dspace.services.factory.DSpaceServicesFactory; - -/* - * - * to do: helpful error messages - can't find mediafilter.cfg - can't - * instantiate filter - bitstream format doesn't exist - * - */ -public class PDFFilter extends MediaFilter { - - private static Logger log = org.apache.logging.log4j.LogManager.getLogger(PDFFilter.class); - - @Override - public String getFilteredName(String oldFilename) { - return oldFilename + ".txt"; - } - - /** - * @return String bundle name - */ - @Override - public String getBundleName() { - return "TEXT"; - } - - /** - * @return String bitstreamformat - */ - @Override - public String getFormatString() { - return "Text"; - } - - /** - * @return String description - */ - @Override - public String getDescription() { - return "Extracted text"; - } - - /** - * @param currentItem item - * @param source source input stream - * @param verbose verbose mode - * @return InputStream the resulting input stream - * @throws Exception if error - */ - @Override - public InputStream getDestinationStream(Item currentItem, InputStream source, boolean verbose) - throws Exception { - ConfigurationService configurationService - = DSpaceServicesFactory.getInstance().getConfigurationService(); - try { - boolean useTemporaryFile = configurationService.getBooleanProperty("pdffilter.largepdfs", false); - - // get input stream from bitstream - // pass to filter, get string back - PDFTextStripper pts = new PDFTextStripper(); - pts.setSortByPosition(true); - PDDocument pdfDoc = null; - Writer writer = null; - File tempTextFile = null; - ByteArrayOutputStream byteStream = null; - - if (useTemporaryFile) { - tempTextFile = File.createTempFile("dspacepdfextract" + source.hashCode(), ".txt"); - tempTextFile.deleteOnExit(); - writer = new OutputStreamWriter(new FileOutputStream(tempTextFile)); - } else { - byteStream = new ByteArrayOutputStream(); - writer = new OutputStreamWriter(byteStream); - } - - try { - pdfDoc = PDDocument.load(source); - pts.writeText(pdfDoc, writer); - } catch (InvalidPasswordException ex) { - log.error("PDF is encrypted. Cannot extract text (item: {})", - () -> currentItem.getHandle()); - return null; - } finally { - try { - if (pdfDoc != null) { - pdfDoc.close(); - } - } catch (Exception e) { - log.error("Error closing PDF file: " + e.getMessage(), e); - } - - try { - writer.close(); - } catch (Exception e) { - log.error("Error closing temporary extract file: " + e.getMessage(), e); - } - } - - if (useTemporaryFile) { - return new FileInputStream(tempTextFile); - } else { - byte[] bytes = byteStream.toByteArray(); - return new ByteArrayInputStream(bytes); - } - } catch (OutOfMemoryError oome) { - log.error("Error parsing PDF document " + oome.getMessage(), oome); - if (!configurationService.getBooleanProperty("pdffilter.skiponmemoryexception", false)) { - throw oome; - } - } - - return null; - } -} diff --git a/dspace-api/src/main/java/org/dspace/app/mediafilter/PoiWordFilter.java b/dspace-api/src/main/java/org/dspace/app/mediafilter/PoiWordFilter.java deleted file mode 100644 index 8c198c4477..0000000000 --- a/dspace-api/src/main/java/org/dspace/app/mediafilter/PoiWordFilter.java +++ /dev/null @@ -1,72 +0,0 @@ -/** - * The contents of this file are subject to the license and copyright - * detailed in the LICENSE and NOTICE files at the root of the source - * tree and available online at - * - * http://www.dspace.org/license/ - */ -package org.dspace.app.mediafilter; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; - -import org.apache.poi.POITextExtractor; -import org.apache.poi.extractor.ExtractorFactory; -import org.apache.poi.openxml4j.exceptions.OpenXML4JException; -import org.apache.xmlbeans.XmlException; -import org.dspace.content.Item; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Extract flat text from Microsoft Word documents (.doc, .docx). - */ -public class PoiWordFilter - extends MediaFilter { - private static final Logger LOG = LoggerFactory.getLogger(PoiWordFilter.class); - - @Override - public String getFilteredName(String oldFilename) { - return oldFilename + ".txt"; - } - - @Override - public String getBundleName() { - return "TEXT"; - } - - @Override - public String getFormatString() { - return "Text"; - } - - @Override - public String getDescription() { - return "Extracted text"; - } - - @Override - public InputStream getDestinationStream(Item currentItem, InputStream source, boolean verbose) - throws Exception { - String text; - try { - // get input stream from bitstream, pass to filter, get string back - POITextExtractor extractor = ExtractorFactory.createExtractor(source); - text = extractor.getText(); - } catch (IOException | OpenXML4JException | XmlException e) { - System.err.format("Invalid File Format: %s%n", e.getMessage()); - LOG.error("Unable to parse the bitstream: ", e); - throw e; - } - - // if verbose flag is set, print out extracted text to STDOUT - if (verbose) { - System.out.println(text); - } - - // return the extracted text as a stream. - return new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8)); - } -} diff --git a/dspace-api/src/main/java/org/dspace/app/mediafilter/PowerPointFilter.java b/dspace-api/src/main/java/org/dspace/app/mediafilter/PowerPointFilter.java deleted file mode 100644 index 86b7096f68..0000000000 --- a/dspace-api/src/main/java/org/dspace/app/mediafilter/PowerPointFilter.java +++ /dev/null @@ -1,113 +0,0 @@ -/** - * The contents of this file are subject to the license and copyright - * detailed in the LICENSE and NOTICE files at the root of the source - * tree and available online at - * - * http://www.dspace.org/license/ - */ -package org.dspace.app.mediafilter; - -import java.io.ByteArrayInputStream; -import java.io.InputStream; - -import org.apache.logging.log4j.Logger; -import org.apache.poi.POITextExtractor; -import org.apache.poi.extractor.ExtractorFactory; -import org.apache.poi.hslf.extractor.PowerPointExtractor; -import org.apache.poi.xslf.extractor.XSLFPowerPointExtractor; -import org.dspace.content.Item; - -/* - * TODO: Allow user to configure extraction of only text or only notes - * - */ -public class PowerPointFilter extends MediaFilter { - - private static Logger log = org.apache.logging.log4j.LogManager.getLogger(PowerPointFilter.class); - - @Override - public String getFilteredName(String oldFilename) { - return oldFilename + ".txt"; - } - - /** - * @return String bundle name - */ - @Override - public String getBundleName() { - return "TEXT"; - } - - /** - * @return String bitstream format - * - * TODO: Check that this is correct - */ - @Override - public String getFormatString() { - return "Text"; - } - - /** - * @return String description - */ - @Override - public String getDescription() { - return "Extracted text"; - } - - /** - * @param currentItem item - * @param source source input stream - * @param verbose verbose mode - * @return InputStream the resulting input stream - * @throws Exception if error - */ - @Override - public InputStream getDestinationStream(Item currentItem, InputStream source, boolean verbose) - throws Exception { - - try { - - String extractedText = null; - new ExtractorFactory(); - POITextExtractor pptExtractor = ExtractorFactory - .createExtractor(source); - - // PowerPoint XML files and legacy format PowerPoint files - // require different classes and APIs for text extraction - - // If this is a PowerPoint XML file, extract accordingly - if (pptExtractor instanceof XSLFPowerPointExtractor) { - - // The true method arguments indicate that text from - // the slides and the notes is desired - extractedText = ((XSLFPowerPointExtractor) pptExtractor) - .getText(true, true); - } else if (pptExtractor instanceof PowerPointExtractor) { // Legacy PowerPoint files - - extractedText = ((PowerPointExtractor) pptExtractor).getText() - + " " + ((PowerPointExtractor) pptExtractor).getNotes(); - - } - if (extractedText != null) { - // if verbose flag is set, print out extracted text - // to STDOUT - if (verbose) { - System.out.println(extractedText); - } - - // generate an input stream with the extracted text - byte[] textBytes = extractedText.getBytes(); - ByteArrayInputStream bais = new ByteArrayInputStream(textBytes); - - return bais; - } - } catch (Exception e) { - log.error("Error filtering bitstream: " + e.getMessage(), e); - throw e; - } - - return null; - } -} diff --git a/dspace-api/src/main/java/org/dspace/app/mediafilter/TikaTextExtractionFilter.java b/dspace-api/src/main/java/org/dspace/app/mediafilter/TikaTextExtractionFilter.java new file mode 100644 index 0000000000..e83bf706ed --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/app/mediafilter/TikaTextExtractionFilter.java @@ -0,0 +1,183 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.mediafilter; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.tika.Tika; +import org.apache.tika.exception.TikaException; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.AutoDetectParser; +import org.apache.tika.sax.BodyContentHandler; +import org.apache.tika.sax.ContentHandlerDecorator; +import org.dspace.content.Item; +import org.dspace.services.ConfigurationService; +import org.dspace.services.factory.DSpaceServicesFactory; +import org.xml.sax.SAXException; + +/** + * Text Extraction media filter which uses Apache Tika to extract text from a large number of file formats (including + * all Microsoft formats, PDF, HTML, Text, etc). For a more complete list of file formats supported by Tika see the + * Tika documentation: https://tika.apache.org/2.3.0/formats.html + */ +public class TikaTextExtractionFilter + extends MediaFilter { + private final static Logger log = LogManager.getLogger(); + + @Override + public String getFilteredName(String oldFilename) { + return oldFilename + ".txt"; + } + + @Override + public String getBundleName() { + return "TEXT"; + } + + @Override + public String getFormatString() { + return "Text"; + } + + @Override + public String getDescription() { + return "Extracted text"; + } + + @Override + public InputStream getDestinationStream(Item currentItem, InputStream source, boolean verbose) + throws Exception { + ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService(); + boolean useTemporaryFile = configurationService.getBooleanProperty("textextractor.use-temp-file", false); + + if (useTemporaryFile) { + // Extract text out of source file using a temp file, returning results as InputStream + return extractUsingTempFile(source, verbose); + } + + // Not using temporary file. We'll use Tika's default in-memory parsing. + // Get maximum characters to extract. Default is 100,000 chars, which is also Tika's default setting. + String extractedText; + int maxChars = configurationService.getIntProperty("textextractor.max-chars", 100000); + try { + // Use Tika to extract text from input. Tika will automatically detect the file type. + Tika tika = new Tika(); + tika.setMaxStringLength(maxChars); // Tell Tika the maximum number of characters to extract + extractedText = tika.parseToString(source); + } catch (IOException e) { + System.err.format("Unable to extract text from bitstream in Item %s%n", currentItem.getID().toString()); + e.printStackTrace(); + log.error("Unable to extract text from bitstream in Item {}", currentItem.getID().toString(), e); + throw e; + } catch (OutOfMemoryError oe) { + System.err.format("OutOfMemoryError occurred when extracting text from bitstream in Item %s. " + + "You may wish to enable 'textextractor.use-temp-file'.%n", currentItem.getID().toString()); + oe.printStackTrace(); + log.error("OutOfMemoryError occurred when extracting text from bitstream in Item {}. " + + "You may wish to enable 'textextractor.use-temp-file'.", currentItem.getID().toString(), oe); + throw oe; + } + + if (StringUtils.isNotEmpty(extractedText)) { + // if verbose flag is set, print out extracted text to STDOUT + if (verbose) { + System.out.println("(Verbose mode) Extracted text:"); + System.out.println(extractedText); + } + + // return the extracted text as a UTF-8 stream. + return new ByteArrayInputStream(extractedText.getBytes(StandardCharsets.UTF_8)); + } + return null; + } + + /** + * Extracts the text out of a given source InputStream, using a temporary file. This decreases the amount of memory + * necessary for text extraction, but can be slower as it requires writing extracted text to a temporary file. + * @param source source InputStream + * @param verbose verbose mode enabled/disabled + * @return InputStream for temporary file containing extracted text + * @throws IOException + * @throws SAXException + * @throws TikaException + */ + private InputStream extractUsingTempFile(InputStream source, boolean verbose) + throws IOException, TikaException, SAXException { + File tempExtractedTextFile = File.createTempFile("dspacetextextract" + source.hashCode(), ".txt"); + + if (verbose) { + System.out.println("(Verbose mode) Extracted text was written to temporary file at " + + tempExtractedTextFile.getAbsolutePath()); + } else { + tempExtractedTextFile.deleteOnExit(); + } + + // Open temp file for writing + try (FileWriter writer = new FileWriter(tempExtractedTextFile, StandardCharsets.UTF_8)) { + // Initialize a custom ContentHandlerDecorator which is a BodyContentHandler. + // This mimics the behavior of Tika().parseToString(), which only extracts text from the body of the file. + // This custom Handler writes any extracted text to the temp file. + ContentHandlerDecorator handler = new BodyContentHandler(new ContentHandlerDecorator() { + /** + * Write all extracted characters directly to the temp file. + */ + @Override + public void characters(char[] ch, int start, int length) throws SAXException { + try { + writer.append(new String(ch), start, length); + } catch (IOException e) { + String errorMsg = String.format("Could not append to temporary file at %s " + + "when performing text extraction", + tempExtractedTextFile.getAbsolutePath()); + log.error(errorMsg, e); + throw new SAXException(errorMsg, e); + } + } + + /** + * Write all ignorable whitespace directly to the temp file. + * This mimics the behaviour of Tika().parseToString() which extracts ignorableWhitespace characters + * (like blank lines, indentations, etc.), so that we get the same extracted text either way. + */ + @Override + public void ignorableWhitespace(char[] ch, int start, int length) throws SAXException { + try { + writer.append(new String(ch), start, length); + } catch (IOException e) { + String errorMsg = String.format("Could not append to temporary file at %s " + + "when performing text extraction", + tempExtractedTextFile.getAbsolutePath()); + log.error(errorMsg, e); + throw new SAXException(errorMsg, e); + } + } + }); + + AutoDetectParser parser = new AutoDetectParser(); + Metadata metadata = new Metadata(); + // parse our source InputStream using the above custom handler + parser.parse(source, handler, metadata); + } + + // At this point, all extracted text is written to our temp file. So, return a FileInputStream for that file + return new FileInputStream(tempExtractedTextFile); + } + + + + +} diff --git a/dspace-api/src/main/java/org/dspace/app/util/Configuration.java b/dspace-api/src/main/java/org/dspace/app/util/Configuration.java index e9b125c41c..e4a59eeb4d 100644 --- a/dspace-api/src/main/java/org/dspace/app/util/Configuration.java +++ b/dspace-api/src/main/java/org/dspace/app/util/Configuration.java @@ -37,6 +37,7 @@ public class Configuration { *
  • {@code --property name} prints the value of the DSpace configuration * property {@code name} to the standard output.
    • *
  • {@code --raw} suppresses parameter substitution in the output.
    • + *
  • {@code --first} print only the first of multiple values.
    • *
  • {@code --help} describes these options.
    • * * If the property does not exist, nothing is written. @@ -51,6 +52,8 @@ public class Configuration { "optional name of the module in which 'property' exists"); options.addOption("r", "raw", false, "do not do property substitution on the value"); + options.addOption("f", "first", false, + "display only the first value of an array property"); options.addOption("?", "Get help"); options.addOption("h", "help", false, "Get help"); @@ -90,19 +93,36 @@ public class Configuration { propNameBuilder.append(cmd.getOptionValue('p')); String propName = propNameBuilder.toString(); - // Print the property's value, if it exists + // Print the property's value(s), if it exists ConfigurationService cfg = DSpaceServicesFactory.getInstance().getConfigurationService(); if (!cfg.hasProperty(propName)) { System.out.println(); } else { - String val; if (cmd.hasOption('r')) { - val = cfg.getPropertyValue(propName).toString(); + // Print "raw" values (without property substitutions) + Object rawValue = cfg.getPropertyValue(propName); + if (rawValue.getClass().isArray()) { + for (Object value : (Object[]) rawValue) { + System.out.println(value.toString()); + if (cmd.hasOption('f')) { + break; // If --first print only one value + } + } + } else { // Not an array + System.out.println(rawValue.toString()); + } } else { - val = cfg.getProperty(propName); + // Print values with property substitutions + String[] values = cfg.getArrayProperty(propName); + for (String value : values) { + System.out.println(value); + if (cmd.hasOption('f')) { + break; // If --first print only one value + } + } } - System.out.println(val); } + System.exit(0); } } diff --git a/dspace-api/src/main/java/org/dspace/authority/orcid/Orcidv3SolrAuthorityImpl.java b/dspace-api/src/main/java/org/dspace/authority/orcid/Orcidv3SolrAuthorityImpl.java index a1c3867fb9..6753a5d113 100644 --- a/dspace-api/src/main/java/org/dspace/authority/orcid/Orcidv3SolrAuthorityImpl.java +++ b/dspace-api/src/main/java/org/dspace/authority/orcid/Orcidv3SolrAuthorityImpl.java @@ -21,7 +21,8 @@ import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpPost; import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.dspace.authority.AuthorityValue; import org.dspace.authority.SolrAuthorityInterface; import org.dspace.external.OrcidRestConnector; @@ -40,7 +41,7 @@ import org.orcid.jaxb.model.v3.release.search.Result; */ public class Orcidv3SolrAuthorityImpl implements SolrAuthorityInterface { - private static Logger log = Logger.getLogger(Orcidv3SolrAuthorityImpl.class); + private final static Logger log = LogManager.getLogger(); private OrcidRestConnector orcidRestConnector; private String OAUTHUrl; diff --git a/dspace-api/src/main/java/org/dspace/authorize/ResourcePolicy.java b/dspace-api/src/main/java/org/dspace/authorize/ResourcePolicy.java index a25a492a3a..954bb96990 100644 --- a/dspace-api/src/main/java/org/dspace/authorize/ResourcePolicy.java +++ b/dspace-api/src/main/java/org/dspace/authorize/ResourcePolicy.java @@ -93,7 +93,7 @@ public class ResourcePolicy implements ReloadableEntity { private String rptype; @Lob - @Type(type = "org.hibernate.type.MaterializedClobType") + @Type(type = "org.dspace.storage.rdbms.hibernate.DatabaseAwareLobType") @Column(name = "rpdescription") private String rpdescription; diff --git a/dspace-api/src/main/java/org/dspace/content/MetadataValue.java b/dspace-api/src/main/java/org/dspace/content/MetadataValue.java index d1b636cdff..9ff3cb9ec2 100644 --- a/dspace-api/src/main/java/org/dspace/content/MetadataValue.java +++ b/dspace-api/src/main/java/org/dspace/content/MetadataValue.java @@ -59,7 +59,7 @@ public class MetadataValue implements ReloadableEntity { * The value of the field */ @Lob - @Type(type = "org.hibernate.type.MaterializedClobType") + @Type(type = "org.dspace.storage.rdbms.hibernate.DatabaseAwareLobType") @Column(name = "text_value") private String value; diff --git a/dspace-api/src/main/java/org/dspace/content/logic/DefaultFilter.java b/dspace-api/src/main/java/org/dspace/content/logic/DefaultFilter.java index c0649e9ea2..490c3949ea 100644 --- a/dspace-api/src/main/java/org/dspace/content/logic/DefaultFilter.java +++ b/dspace-api/src/main/java/org/dspace/content/logic/DefaultFilter.java @@ -7,7 +7,8 @@ */ package org.dspace.content.logic; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.dspace.content.Item; import org.dspace.core.Context; @@ -21,7 +22,7 @@ import org.dspace.core.Context; */ public class DefaultFilter implements Filter { private LogicalStatement statement; - private static Logger log = Logger.getLogger(Filter.class); + private final static Logger log = LogManager.getLogger(); /** * Set statement from Spring configuration in item-filters.xml diff --git a/dspace-api/src/main/java/org/dspace/content/logic/condition/InCommunityCondition.java b/dspace-api/src/main/java/org/dspace/content/logic/condition/InCommunityCondition.java index b9c1d15d2a..9f588f9c3b 100644 --- a/dspace-api/src/main/java/org/dspace/content/logic/condition/InCommunityCondition.java +++ b/dspace-api/src/main/java/org/dspace/content/logic/condition/InCommunityCondition.java @@ -10,7 +10,8 @@ package org.dspace.content.logic.condition; import java.sql.SQLException; import java.util.List; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.dspace.content.Collection; import org.dspace.content.Community; import org.dspace.content.DSpaceObject; @@ -26,7 +27,7 @@ import org.dspace.core.Context; * @version $Revision$ */ public class InCommunityCondition extends AbstractCondition { - private static Logger log = Logger.getLogger(InCommunityCondition.class); + private final static Logger log = LogManager.getLogger(); /** * Return true if item is in one of the specified collections diff --git a/dspace-api/src/main/java/org/dspace/content/logic/condition/IsWithdrawnCondition.java b/dspace-api/src/main/java/org/dspace/content/logic/condition/IsWithdrawnCondition.java index 6475ef09e2..6424e6f35f 100644 --- a/dspace-api/src/main/java/org/dspace/content/logic/condition/IsWithdrawnCondition.java +++ b/dspace-api/src/main/java/org/dspace/content/logic/condition/IsWithdrawnCondition.java @@ -7,7 +7,8 @@ */ package org.dspace.content.logic.condition; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.dspace.content.Item; import org.dspace.content.logic.LogicalStatementException; import org.dspace.core.Context; @@ -19,7 +20,7 @@ import org.dspace.core.Context; * @version $Revision$ */ public class IsWithdrawnCondition extends AbstractCondition { - private static Logger log = Logger.getLogger(IsWithdrawnCondition.class); + private final static Logger log = LogManager.getLogger(); /** * Return true if item is withdrawn diff --git a/dspace-api/src/main/java/org/dspace/content/logic/condition/MetadataValueMatchCondition.java b/dspace-api/src/main/java/org/dspace/content/logic/condition/MetadataValueMatchCondition.java index d9c774485a..4e30c75a2a 100644 --- a/dspace-api/src/main/java/org/dspace/content/logic/condition/MetadataValueMatchCondition.java +++ b/dspace-api/src/main/java/org/dspace/content/logic/condition/MetadataValueMatchCondition.java @@ -11,7 +11,8 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.dspace.content.Item; import org.dspace.content.MetadataValue; import org.dspace.content.logic.LogicalStatementException; @@ -26,7 +27,7 @@ import org.dspace.core.Context; */ public class MetadataValueMatchCondition extends AbstractCondition { - private static Logger log = Logger.getLogger(MetadataValueMatchCondition.class); + private final static Logger log = LogManager.getLogger(); /** * Return true if any value for a specified field in the item matches a specified regex pattern diff --git a/dspace-api/src/main/java/org/dspace/content/logic/condition/MetadataValuesMatchCondition.java b/dspace-api/src/main/java/org/dspace/content/logic/condition/MetadataValuesMatchCondition.java index df9cbfbf1d..74ccfa4ca8 100644 --- a/dspace-api/src/main/java/org/dspace/content/logic/condition/MetadataValuesMatchCondition.java +++ b/dspace-api/src/main/java/org/dspace/content/logic/condition/MetadataValuesMatchCondition.java @@ -11,7 +11,8 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.dspace.content.Item; import org.dspace.content.MetadataValue; import org.dspace.content.logic.LogicalStatementException; @@ -26,7 +27,7 @@ import org.dspace.core.Context; */ public class MetadataValuesMatchCondition extends AbstractCondition { - private static Logger log = Logger.getLogger(MetadataValuesMatchCondition.class); + private final static Logger log = LogManager.getLogger(); /** * Return true if any value for a specified field in the item matches any of the specified regex patterns diff --git a/dspace-api/src/main/java/org/dspace/content/logic/condition/ReadableByGroupCondition.java b/dspace-api/src/main/java/org/dspace/content/logic/condition/ReadableByGroupCondition.java index e76772803c..65f9925222 100644 --- a/dspace-api/src/main/java/org/dspace/content/logic/condition/ReadableByGroupCondition.java +++ b/dspace-api/src/main/java/org/dspace/content/logic/condition/ReadableByGroupCondition.java @@ -10,7 +10,8 @@ package org.dspace.content.logic.condition; import java.sql.SQLException; import java.util.List; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.dspace.authorize.ResourcePolicy; import org.dspace.authorize.factory.AuthorizeServiceFactory; import org.dspace.authorize.service.AuthorizeService; @@ -27,7 +28,7 @@ import org.dspace.core.Context; * @version $Revision$ */ public class ReadableByGroupCondition extends AbstractCondition { - private static Logger log = Logger.getLogger(ReadableByGroupCondition.class); + private final static Logger log = LogManager.getLogger(); // Authorize service AuthorizeService authorizeService = AuthorizeServiceFactory.getInstance().getAuthorizeService(); diff --git a/dspace-api/src/main/java/org/dspace/core/Email.java b/dspace-api/src/main/java/org/dspace/core/Email.java index c0d191caf5..6db27c9e4f 100644 --- a/dspace-api/src/main/java/org/dspace/core/Email.java +++ b/dspace-api/src/main/java/org/dspace/core/Email.java @@ -152,14 +152,14 @@ public class Email { private static final String RESOURCE_REPOSITORY_NAME = "Email"; private static final Properties VELOCITY_PROPERTIES = new Properties(); static { - VELOCITY_PROPERTIES.put(Velocity.RESOURCE_LOADER, "string"); - VELOCITY_PROPERTIES.put("string.resource.loader.description", + VELOCITY_PROPERTIES.put(Velocity.RESOURCE_LOADERS, "string"); + VELOCITY_PROPERTIES.put("resource.loader.string.description", "Velocity StringResource loader"); - VELOCITY_PROPERTIES.put("string.resource.loader.class", + VELOCITY_PROPERTIES.put("resource.loader.string.class", StringResourceLoader.class.getName()); - VELOCITY_PROPERTIES.put("string.resource.loader.repository.name", + VELOCITY_PROPERTIES.put("resource.loader.string.repository.name", RESOURCE_REPOSITORY_NAME); - VELOCITY_PROPERTIES.put("string.resource.loader.repository.static", + VELOCITY_PROPERTIES.put("resource.loader.string.repository.static", "false"); } diff --git a/dspace-api/src/main/java/org/dspace/discovery/indexobject/IndexFactoryImpl.java b/dspace-api/src/main/java/org/dspace/discovery/indexobject/IndexFactoryImpl.java index 8660bbebc7..3c93e1c522 100644 --- a/dspace-api/src/main/java/org/dspace/discovery/indexobject/IndexFactoryImpl.java +++ b/dspace-api/src/main/java/org/dspace/discovery/indexobject/IndexFactoryImpl.java @@ -95,7 +95,6 @@ public abstract class IndexFactoryImpl implements 100000); // Use Tika's Text parser as the streams are always from the TEXT bundle (i.e. already extracted text) - // TODO: We may wish to consider using Tika to extract the text in the future. TextAndCSVParser tikaParser = new TextAndCSVParser(); BodyContentHandler tikaHandler = new BodyContentHandler(charLimit); Metadata tikaMetadata = new Metadata(); diff --git a/dspace-api/src/main/java/org/dspace/google/GoogleAsyncEventListener.java b/dspace-api/src/main/java/org/dspace/google/GoogleAsyncEventListener.java index cf5c40976d..9cca3777ef 100644 --- a/dspace-api/src/main/java/org/dspace/google/GoogleAsyncEventListener.java +++ b/dspace-api/src/main/java/org/dspace/google/GoogleAsyncEventListener.java @@ -31,7 +31,8 @@ import org.apache.http.client.methods.HttpPost; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.dspace.content.factory.ContentServiceFactory; import org.dspace.core.Constants; import org.dspace.core.Context; @@ -55,7 +56,7 @@ public class GoogleAsyncEventListener extends AbstractUsageEventListener { // 20 is the event max set by the GA API private static final int GA_MAX_EVENTS = 20; private static final String ANALYTICS_BATCH_ENDPOINT = "https://www.google-analytics.com/batch"; - private static Logger log = Logger.getLogger(GoogleAsyncEventListener.class); + private final static Logger log = LogManager.getLogger(); private static String analyticsKey; private static CloseableHttpClient httpclient; private static Buffer buffer; diff --git a/dspace-api/src/main/java/org/dspace/storage/rdbms/EntityTypeServiceInitializer.java b/dspace-api/src/main/java/org/dspace/storage/rdbms/EntityTypeServiceInitializer.java index ebf790900b..e0e41516d0 100644 --- a/dspace-api/src/main/java/org/dspace/storage/rdbms/EntityTypeServiceInitializer.java +++ b/dspace-api/src/main/java/org/dspace/storage/rdbms/EntityTypeServiceInitializer.java @@ -49,6 +49,16 @@ public class EntityTypeServiceInitializer implements Callback { } } + /** + * The callback name, Flyway will use this to sort the callbacks alphabetically before executing them + * @return The callback name + */ + @Override + public String getCallbackName() { + // Return class name only (not prepended by package) + return EntityTypeServiceInitializer.class.getSimpleName(); + } + @Override public boolean supports(Event event, org.flywaydb.core.api.callback.Context context) { // Must run AFTER all migrations complete, since it is dependent on Hibernate diff --git a/dspace-api/src/main/java/org/dspace/storage/rdbms/GroupServiceInitializer.java b/dspace-api/src/main/java/org/dspace/storage/rdbms/GroupServiceInitializer.java index 7338dd75bc..54498a1c64 100644 --- a/dspace-api/src/main/java/org/dspace/storage/rdbms/GroupServiceInitializer.java +++ b/dspace-api/src/main/java/org/dspace/storage/rdbms/GroupServiceInitializer.java @@ -51,6 +51,16 @@ public class GroupServiceInitializer implements Callback { } + /** + * The callback name, Flyway will use this to sort the callbacks alphabetically before executing them + * @return The callback name + */ + @Override + public String getCallbackName() { + // Return class name only (not prepended by package) + return GroupServiceInitializer.class.getSimpleName(); + } + /** * Events supported by this callback. * @param event Flyway event diff --git a/dspace-api/src/main/java/org/dspace/storage/rdbms/PostgreSQLCryptoChecker.java b/dspace-api/src/main/java/org/dspace/storage/rdbms/PostgreSQLCryptoChecker.java index 5798f4254c..5459cc3cc3 100644 --- a/dspace-api/src/main/java/org/dspace/storage/rdbms/PostgreSQLCryptoChecker.java +++ b/dspace-api/src/main/java/org/dspace/storage/rdbms/PostgreSQLCryptoChecker.java @@ -97,6 +97,16 @@ public class PostgreSQLCryptoChecker implements Callback { } } + /** + * The callback name, Flyway will use this to sort the callbacks alphabetically before executing them + * @return The callback name + */ + @Override + public String getCallbackName() { + // Return class name only (not prepended by package) + return PostgreSQLCryptoChecker.class.getSimpleName(); + } + /** * Events supported by this callback. * @param event Flyway event diff --git a/dspace-api/src/main/java/org/dspace/storage/rdbms/RegistryUpdater.java b/dspace-api/src/main/java/org/dspace/storage/rdbms/RegistryUpdater.java index ae8be0988a..c2a9b8a848 100644 --- a/dspace-api/src/main/java/org/dspace/storage/rdbms/RegistryUpdater.java +++ b/dspace-api/src/main/java/org/dspace/storage/rdbms/RegistryUpdater.java @@ -101,6 +101,16 @@ public class RegistryUpdater implements Callback { } + /** + * The callback name, Flyway will use this to sort the callbacks alphabetically before executing them + * @return The callback name + */ + @Override + public String getCallbackName() { + // Return class name only (not prepended by package) + return RegistryUpdater.class.getSimpleName(); + } + /** * Events supported by this callback. * @param event Flyway event diff --git a/dspace-api/src/main/java/org/dspace/storage/rdbms/SiteServiceInitializer.java b/dspace-api/src/main/java/org/dspace/storage/rdbms/SiteServiceInitializer.java index 26e76804e1..872a633146 100644 --- a/dspace-api/src/main/java/org/dspace/storage/rdbms/SiteServiceInitializer.java +++ b/dspace-api/src/main/java/org/dspace/storage/rdbms/SiteServiceInitializer.java @@ -73,6 +73,16 @@ public class SiteServiceInitializer implements Callback { } + /** + * The callback name, Flyway will use this to sort the callbacks alphabetically before executing them + * @return The callback name + */ + @Override + public String getCallbackName() { + // Return class name only (not prepended by package) + return SiteServiceInitializer.class.getSimpleName(); + } + /** * Events supported by this callback. * @param event Flyway event diff --git a/dspace-api/src/main/java/org/dspace/storage/rdbms/hibernate/DatabaseAwareLobType.java b/dspace-api/src/main/java/org/dspace/storage/rdbms/hibernate/DatabaseAwareLobType.java new file mode 100644 index 0000000000..95939f9902 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/storage/rdbms/hibernate/DatabaseAwareLobType.java @@ -0,0 +1,57 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.storage.rdbms.hibernate; + +import org.apache.commons.lang.StringUtils; +import org.dspace.services.ConfigurationService; +import org.dspace.services.factory.DSpaceServicesFactory; +import org.hibernate.type.AbstractSingleColumnStandardBasicType; +import org.hibernate.type.descriptor.java.StringTypeDescriptor; +import org.hibernate.type.descriptor.sql.ClobTypeDescriptor; +import org.hibernate.type.descriptor.sql.LongVarcharTypeDescriptor; +import org.hibernate.type.descriptor.sql.SqlTypeDescriptor; + +/** + * A Hibernate @Type used to properly support the CLOB in both Postgres and Oracle. + * PostgreSQL doesn't have a CLOB type, instead it's a TEXT field. + * Normally, you'd use org.hibernate.type.TextType to support TEXT, but that won't work for Oracle. + * https://github.com/hibernate/hibernate-orm/blob/5.6/hibernate-core/src/main/java/org/hibernate/type/TextType.java + * + * This Type checks if we are using PostgreSQL. + * If so, it configures Hibernate to map CLOB to LongVarChar (same as org.hibernate.type.TextType) + * If not, it uses default CLOB (which works for other databases). + */ +public class DatabaseAwareLobType extends AbstractSingleColumnStandardBasicType { + + public static final DatabaseAwareLobType INSTANCE = new DatabaseAwareLobType(); + + public DatabaseAwareLobType() { + super( getDbDescriptor(), StringTypeDescriptor.INSTANCE ); + } + + public static SqlTypeDescriptor getDbDescriptor() { + if ( isPostgres() ) { + return LongVarcharTypeDescriptor.INSTANCE; + } else { + return ClobTypeDescriptor.DEFAULT; + } + } + + private static boolean isPostgres() { + ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService(); + String dbDialect = configurationService.getProperty("db.dialect"); + + return StringUtils.containsIgnoreCase(dbDialect, "PostgreSQL"); + } + + @Override + public String getName() { + return "database_aware_lob"; + } +} + diff --git a/dspace-api/src/main/java/org/dspace/storage/rdbms/hibernate/postgres/DSpacePostgreSQL82Dialect.java b/dspace-api/src/main/java/org/dspace/storage/rdbms/hibernate/postgres/DSpacePostgreSQL82Dialect.java deleted file mode 100644 index 2701c22fd2..0000000000 --- a/dspace-api/src/main/java/org/dspace/storage/rdbms/hibernate/postgres/DSpacePostgreSQL82Dialect.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * The contents of this file are subject to the license and copyright - * detailed in the LICENSE and NOTICE files at the root of the source - * tree and available online at - * - * http://www.dspace.org/license/ - */ -package org.dspace.storage.rdbms.hibernate.postgres; - -import java.sql.Types; - -import org.hibernate.dialect.PostgreSQL82Dialect; -import org.hibernate.service.ServiceRegistry; -import org.hibernate.type.PostgresUUIDType; -import org.hibernate.type.descriptor.sql.LongVarcharTypeDescriptor; -import org.hibernate.type.descriptor.sql.SqlTypeDescriptor; - -/** - * UUID's are not supported by default in hibernate due to differences in the database in order to fix this a custom - * sql dialect is needed. - * Source: https://forum.hibernate.org/viewtopic.php?f=1&t=1014157 - * - * @author kevinvandevelde at atmire.com - */ -public class DSpacePostgreSQL82Dialect extends PostgreSQL82Dialect { - @Override - public void contributeTypes(final org.hibernate.boot.model.TypeContributions typeContributions, - final ServiceRegistry serviceRegistry) { - super.contributeTypes(typeContributions, serviceRegistry); - typeContributions.contributeType(new InternalPostgresUUIDType()); - } - - @Override - protected void registerHibernateType(int code, String name) { - super.registerHibernateType(code, name); - } - - protected static class InternalPostgresUUIDType extends PostgresUUIDType { - - @Override - protected boolean registerUnderJavaType() { - return true; - } - } - - /** - * Override is needed to properly support the CLOB on metadatavalue in Postgres and Oracle. - * - * @param sqlCode {@linkplain java.sql.Types JDBC type-code} for the column mapped by this type. - * @return Descriptor for the SQL/JDBC side of a value mapping. - */ - @Override - public SqlTypeDescriptor getSqlTypeDescriptorOverride(int sqlCode) { - SqlTypeDescriptor descriptor; - switch (sqlCode) { - case Types.CLOB: { - descriptor = LongVarcharTypeDescriptor.INSTANCE; - break; - } - default: { - descriptor = super.getSqlTypeDescriptorOverride(sqlCode); - break; - } - } - return descriptor; - } -} diff --git a/dspace-api/src/main/java/org/dspace/storage/rdbms/migration/MigrationUtils.java b/dspace-api/src/main/java/org/dspace/storage/rdbms/migration/MigrationUtils.java index 624d0cb55a..842fc15e16 100644 --- a/dspace-api/src/main/java/org/dspace/storage/rdbms/migration/MigrationUtils.java +++ b/dspace-api/src/main/java/org/dspace/storage/rdbms/migration/MigrationUtils.java @@ -86,10 +86,11 @@ public class MigrationUtils { cascade = true; break; case "h2": - // In H2, constraints are listed in the "information_schema.constraints" table + // In H2, column constraints are listed in the "INFORMATION_SCHEMA.KEY_COLUMN_USAGE" table constraintNameSQL = "SELECT DISTINCT CONSTRAINT_NAME " + - "FROM information_schema.constraints " + - "WHERE table_name = ? AND column_list = ?"; + "FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE " + + "WHERE TABLE_NAME = ? AND COLUMN_NAME = ?"; + cascade = true; break; default: throw new SQLException("DBMS " + dbtype + " is unsupported in this migration."); diff --git a/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V1.4__Upgrade_to_DSpace_1.4_schema.sql b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V1.4__Upgrade_to_DSpace_1.4_schema.sql index e00a651626..62d12fe5ce 100644 --- a/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V1.4__Upgrade_to_DSpace_1.4_schema.sql +++ b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V1.4__Upgrade_to_DSpace_1.4_schema.sql @@ -245,13 +245,13 @@ insert into most_recent_checksum ) select bitstream.bitstream_id, - '1', + true, CASE WHEN bitstream.checksum IS NULL THEN '' ELSE bitstream.checksum END, CASE WHEN bitstream.checksum IS NULL THEN '' ELSE bitstream.checksum END, FORMATDATETIME(NOW(),'DD-MM-RRRR HH24:MI:SS'), FORMATDATETIME(NOW(),'DD-MM-RRRR HH24:MI:SS'), CASE WHEN bitstream.checksum_algorithm IS NULL THEN 'MD5' ELSE bitstream.checksum_algorithm END, - '1' + true from bitstream; -- Update all the deleted checksums @@ -263,7 +263,7 @@ update most_recent_checksum set to_be_processed = 0 where most_recent_checksum.bitstream_id in ( select bitstream_id -from bitstream where deleted = '1' ); +from bitstream where deleted = true ); -- this will insert into history table -- for the initial start diff --git a/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V5.0_2014.09.26__DS-1582_Metadata_For_All_Objects.sql b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V5.0_2014.09.26__DS-1582_Metadata_For_All_Objects.sql index 87551bdf4e..cd908279f1 100644 --- a/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V5.0_2014.09.26__DS-1582_Metadata_For_All_Objects.sql +++ b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V5.0_2014.09.26__DS-1582_Metadata_For_All_Objects.sql @@ -36,7 +36,7 @@ alter table metadatavalue alter column resource_id set not null; INSERT INTO metadatavalue (metadata_value_id, resource_id, resource_type_id, metadata_field_id, text_value, text_lang, place) SELECT -metadatavalue_seq.nextval as metadata_value_id, +NEXT VALUE FOR metadatavalue_seq as metadata_value_id, community_id AS resource_id, 4 AS resource_type_id, (select metadata_field_id from metadatafieldregistry where metadata_schema_id=(select metadata_schema_id from metadataschemaregistry where short_id='dc') and element = 'description' and qualifier is null) AS metadata_field_id, @@ -47,7 +47,7 @@ FROM community where not introductory_text is null; INSERT INTO metadatavalue (metadata_value_id, resource_id, resource_type_id, metadata_field_id, text_value, text_lang, place) SELECT -metadatavalue_seq.nextval as metadata_value_id, +NEXT VALUE FOR metadatavalue_seq as metadata_value_id, community_id AS resource_id, 4 AS resource_type_id, (select metadata_field_id from metadatafieldregistry where metadata_schema_id=(select metadata_schema_id from metadataschemaregistry where short_id='dc') and element = 'description' and qualifier = 'abstract') AS metadata_field_id, @@ -58,7 +58,7 @@ FROM community where not short_description is null; INSERT INTO metadatavalue (metadata_value_id, resource_id, resource_type_id, metadata_field_id, text_value, text_lang, place) SELECT -metadatavalue_seq.nextval as metadata_value_id, +NEXT VALUE FOR metadatavalue_seq as metadata_value_id, community_id AS resource_id, 4 AS resource_type_id, (select metadata_field_id from metadatafieldregistry where metadata_schema_id=(select metadata_schema_id from metadataschemaregistry where short_id='dc') and element = 'description' and qualifier = 'tableofcontents') AS metadata_field_id, @@ -69,7 +69,7 @@ FROM community where not side_bar_text is null; INSERT INTO metadatavalue (metadata_value_id, resource_id, resource_type_id, metadata_field_id, text_value, text_lang, place) SELECT -metadatavalue_seq.nextval as metadata_value_id, +NEXT VALUE FOR metadatavalue_seq as metadata_value_id, community_id AS resource_id, 4 AS resource_type_id, (select metadata_field_id from metadatafieldregistry where metadata_schema_id=(select metadata_schema_id from metadataschemaregistry where short_id='dc') and element = 'rights' and qualifier is null) AS metadata_field_id, @@ -80,7 +80,7 @@ FROM community where not copyright_text is null; INSERT INTO metadatavalue (metadata_value_id, resource_id, resource_type_id, metadata_field_id, text_value, text_lang, place) SELECT -metadatavalue_seq.nextval as metadata_value_id, +NEXT VALUE FOR metadatavalue_seq as metadata_value_id, community_id AS resource_id, 4 AS resource_type_id, (select metadata_field_id from metadatafieldregistry where metadata_schema_id=(select metadata_schema_id from metadataschemaregistry where short_id='dc') and element = 'title' and qualifier is null) AS metadata_field_id, @@ -104,7 +104,7 @@ alter table community drop column name; INSERT INTO metadatavalue (metadata_value_id, resource_id, resource_type_id, metadata_field_id, text_value, text_lang, place) SELECT -metadatavalue_seq.nextval as metadata_value_id, +NEXT VALUE FOR metadatavalue_seq as metadata_value_id, collection_id AS resource_id, 3 AS resource_type_id, (select metadata_field_id from metadatafieldregistry where metadata_schema_id=(select metadata_schema_id from metadataschemaregistry where short_id='dc') and element = 'description' and qualifier is null) AS metadata_field_id, @@ -115,7 +115,7 @@ FROM collection where not introductory_text is null; INSERT INTO metadatavalue (metadata_value_id, resource_id, resource_type_id, metadata_field_id, text_value, text_lang, place) SELECT -metadatavalue_seq.nextval as metadata_value_id, +NEXT VALUE FOR metadatavalue_seq as metadata_value_id, collection_id AS resource_id, 3 AS resource_type_id, (select metadata_field_id from metadatafieldregistry where metadata_schema_id=(select metadata_schema_id from metadataschemaregistry where short_id='dc') and element = 'description' and qualifier = 'abstract') AS metadata_field_id, @@ -126,7 +126,7 @@ FROM collection where not short_description is null; INSERT INTO metadatavalue (metadata_value_id, resource_id, resource_type_id, metadata_field_id, text_value, text_lang, place) SELECT -metadatavalue_seq.nextval as metadata_value_id, +NEXT VALUE FOR metadatavalue_seq as metadata_value_id, collection_id AS resource_id, 3 AS resource_type_id, (select metadata_field_id from metadatafieldregistry where metadata_schema_id=(select metadata_schema_id from metadataschemaregistry where short_id='dc') and element = 'description' and qualifier = 'tableofcontents') AS metadata_field_id, @@ -137,7 +137,7 @@ FROM collection where not side_bar_text is null; INSERT INTO metadatavalue (metadata_value_id, resource_id, resource_type_id, metadata_field_id, text_value, text_lang, place) SELECT -metadatavalue_seq.nextval as metadata_value_id, +NEXT VALUE FOR metadatavalue_seq as metadata_value_id, collection_id AS resource_id, 3 AS resource_type_id, (select metadata_field_id from metadatafieldregistry where metadata_schema_id=(select metadata_schema_id from metadataschemaregistry where short_id='dc') and element = 'rights' and qualifier is null) AS metadata_field_id, @@ -148,7 +148,7 @@ FROM collection where not copyright_text is null; INSERT INTO metadatavalue (metadata_value_id, resource_id, resource_type_id, metadata_field_id, text_value, text_lang, place) SELECT -metadatavalue_seq.nextval as metadata_value_id, +NEXT VALUE FOR metadatavalue_seq as metadata_value_id, collection_id AS resource_id, 3 AS resource_type_id, (select metadata_field_id from metadatafieldregistry where metadata_schema_id=(select metadata_schema_id from metadataschemaregistry where short_id='dc') and element = 'title' and qualifier is null) AS metadata_field_id, @@ -159,7 +159,7 @@ FROM collection where not name is null; INSERT INTO metadatavalue (metadata_value_id, resource_id, resource_type_id, metadata_field_id, text_value, text_lang, place) SELECT -metadatavalue_seq.nextval as metadata_value_id, +NEXT VALUE FOR metadatavalue_seq as metadata_value_id, collection_id AS resource_id, 3 AS resource_type_id, (select metadata_field_id from metadatafieldregistry where metadata_schema_id=(select metadata_schema_id from metadataschemaregistry where short_id='dc') and element = 'provenance' and qualifier is null) AS metadata_field_id, @@ -170,7 +170,7 @@ FROM collection where not provenance_description is null; INSERT INTO metadatavalue (metadata_value_id, resource_id, resource_type_id, metadata_field_id, text_value, text_lang, place) SELECT -metadatavalue_seq.nextval as metadata_value_id, +NEXT VALUE FOR metadatavalue_seq as metadata_value_id, collection_id AS resource_id, 3 AS resource_type_id, (select metadata_field_id from metadatafieldregistry where metadata_schema_id=(select metadata_schema_id from metadataschemaregistry where short_id='dc') and element = 'rights' and qualifier = 'license') AS metadata_field_id, @@ -194,7 +194,7 @@ alter table collection drop column provenance_description; INSERT INTO metadatavalue (metadata_value_id, resource_id, resource_type_id, metadata_field_id, text_value, text_lang, place) SELECT -metadatavalue_seq.nextval as metadata_value_id, +NEXT VALUE FOR metadatavalue_seq as metadata_value_id, bundle_id AS resource_id, 1 AS resource_type_id, (select metadata_field_id from metadatafieldregistry where metadata_schema_id=(select metadata_schema_id from metadataschemaregistry where short_id='dc') and element = 'title' and qualifier is null) AS metadata_field_id, @@ -214,7 +214,7 @@ alter table bundle drop column name; INSERT INTO metadatavalue (metadata_value_id, resource_id, resource_type_id, metadata_field_id, text_value, text_lang, place) SELECT -metadatavalue_seq.nextval as metadata_value_id, +NEXT VALUE FOR metadatavalue_seq as metadata_value_id, bitstream_id AS resource_id, 0 AS resource_type_id, (select metadata_field_id from metadatafieldregistry where metadata_schema_id=(select metadata_schema_id from metadataschemaregistry where short_id='dc') and element = 'title' and qualifier is null) AS metadata_field_id, @@ -225,7 +225,7 @@ FROM bitstream where not name is null; INSERT INTO metadatavalue (metadata_value_id, resource_id, resource_type_id, metadata_field_id, text_value, text_lang, place) SELECT -metadatavalue_seq.nextval as metadata_value_id, +NEXT VALUE FOR metadatavalue_seq as metadata_value_id, bitstream_id AS resource_id, 0 AS resource_type_id, (select metadata_field_id from metadatafieldregistry where metadata_schema_id=(select metadata_schema_id from metadataschemaregistry where short_id='dc') and element = 'description' and qualifier is null) AS metadata_field_id, @@ -236,7 +236,7 @@ FROM bitstream where not description is null; INSERT INTO metadatavalue (metadata_value_id, resource_id, resource_type_id, metadata_field_id, text_value, text_lang, place) SELECT -metadatavalue_seq.nextval as metadata_value_id, +NEXT VALUE FOR metadatavalue_seq as metadata_value_id, bitstream_id AS resource_id, 0 AS resource_type_id, (select metadata_field_id from metadatafieldregistry where metadata_schema_id=(select metadata_schema_id from metadataschemaregistry where short_id='dc') and element = 'format' and qualifier is null) AS metadata_field_id, @@ -247,7 +247,7 @@ FROM bitstream where not user_format_description is null; INSERT INTO metadatavalue (metadata_value_id, resource_id, resource_type_id, metadata_field_id, text_value, text_lang, place) SELECT -metadatavalue_seq.nextval as metadata_value_id, +NEXT VALUE FOR metadatavalue_seq as metadata_value_id, bitstream_id AS resource_id, 0 AS resource_type_id, (select metadata_field_id from metadatafieldregistry where metadata_schema_id=(select metadata_schema_id from metadataschemaregistry where short_id='dc') and element = 'source' and qualifier is null) AS metadata_field_id, @@ -269,7 +269,7 @@ alter table bitstream drop column source; INSERT INTO metadatavalue (metadata_value_id, resource_id, resource_type_id, metadata_field_id, text_value, text_lang, place) SELECT -metadatavalue_seq.nextval as metadata_value_id, +NEXT VALUE FOR metadatavalue_seq as metadata_value_id, eperson_group_id AS resource_id, 6 AS resource_type_id, (select metadata_field_id from metadatafieldregistry where metadata_schema_id=(select metadata_schema_id from metadataschemaregistry where short_id='dc') and element = 'title' and qualifier is null) AS metadata_field_id, @@ -288,7 +288,7 @@ alter table epersongroup drop column name; INSERT INTO metadatavalue (metadata_value_id, resource_id, resource_type_id, metadata_field_id, text_value, text_lang, place) SELECT -metadatavalue_seq.nextval as metadata_value_id, +NEXT VALUE FOR metadatavalue_seq as metadata_value_id, eperson_id AS resource_id, 7 AS resource_type_id, (select metadata_field_id from metadatafieldregistry where metadata_schema_id=(select metadata_schema_id from metadataschemaregistry where short_id='eperson') and element = 'firstname' and qualifier is null) AS metadata_field_id, @@ -299,7 +299,7 @@ FROM eperson where not firstname is null; INSERT INTO metadatavalue (metadata_value_id, resource_id, resource_type_id, metadata_field_id, text_value, text_lang, place) SELECT -metadatavalue_seq.nextval as metadata_value_id, +NEXT VALUE FOR metadatavalue_seq as metadata_value_id, eperson_id AS resource_id, 7 AS resource_type_id, (select metadata_field_id from metadatafieldregistry where metadata_schema_id=(select metadata_schema_id from metadataschemaregistry where short_id='eperson') and element = 'lastname' and qualifier is null) AS metadata_field_id, @@ -310,7 +310,7 @@ FROM eperson where not lastname is null; INSERT INTO metadatavalue (metadata_value_id, resource_id, resource_type_id, metadata_field_id, text_value, text_lang, place) SELECT -metadatavalue_seq.nextval as metadata_value_id, +NEXT VALUE FOR metadatavalue_seq as metadata_value_id, eperson_id AS resource_id, 7 AS resource_type_id, (select metadata_field_id from metadatafieldregistry where metadata_schema_id=(select metadata_schema_id from metadataschemaregistry where short_id='eperson') and element = 'phone' and qualifier is null) AS metadata_field_id, @@ -321,7 +321,7 @@ FROM eperson where not phone is null; INSERT INTO metadatavalue (metadata_value_id, resource_id, resource_type_id, metadata_field_id, text_value, text_lang, place) SELECT -metadatavalue_seq.nextval as metadata_value_id, +NEXT VALUE FOR metadatavalue_seq as metadata_value_id, eperson_id AS resource_id, 7 AS resource_type_id, (select metadata_field_id from metadatafieldregistry where metadata_schema_id=(select metadata_schema_id from metadataschemaregistry where short_id='eperson') and element = 'language' and qualifier is null) AS metadata_field_id, diff --git a/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V5.6_2016.08.23__DS-3097.sql b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V5.6_2016.08.23__DS-3097.sql index 2e09b807de..0bd68c5201 100644 --- a/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V5.6_2016.08.23__DS-3097.sql +++ b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V5.6_2016.08.23__DS-3097.sql @@ -14,11 +14,11 @@ UPDATE resourcepolicy SET action_id = 12 where action_id = 0 and resource_type_i SELECT bundle2bitstream.bitstream_id FROM bundle2bitstream LEFT JOIN item2bundle ON bundle2bitstream.bundle_id = item2bundle.bundle_id LEFT JOIN item ON item2bundle.item_id = item.item_id - WHERE item.withdrawn = 1 + WHERE item.withdrawn = true ); UPDATE resourcepolicy SET action_id = 12 where action_id = 0 and resource_type_id = 1 and resource_id in ( SELECT item2bundle.bundle_id FROM item2bundle LEFT JOIN item ON item2bundle.item_id = item.item_id - WHERE item.withdrawn = 1 + WHERE item.withdrawn = true ); \ No newline at end of file diff --git a/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V6.0_2016.04.14__DS-3125-fix-bundle-bitstream-delete-rights.sql b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V6.0_2016.04.14__DS-3125-fix-bundle-bitstream-delete-rights.sql index 1c98ceef2a..1ee23246ea 100644 --- a/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V6.0_2016.04.14__DS-3125-fix-bundle-bitstream-delete-rights.sql +++ b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V6.0_2016.04.14__DS-3125-fix-bundle-bitstream-delete-rights.sql @@ -17,7 +17,7 @@ INSERT INTO resourcepolicy (policy_id, resource_type_id, resource_id, action_id, start_date, end_date, rpname, rptype, rpdescription, eperson_id, epersongroup_id, dspace_object) SELECT -resourcepolicy_seq.nextval AS policy_id, +NEXT VALUE FOR resourcepolicy_seq AS policy_id, resource_type_id, resource_id, -- Insert the Constants.DELETE action diff --git a/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V6.0_2016.08.23__DS-3097.sql b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V6.0_2016.08.23__DS-3097.sql index e1220c8c7c..5bb59970c5 100644 --- a/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V6.0_2016.08.23__DS-3097.sql +++ b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V6.0_2016.08.23__DS-3097.sql @@ -14,11 +14,11 @@ UPDATE resourcepolicy SET action_id = 12 where action_id = 0 and dspace_object i SELECT bundle2bitstream.bitstream_id FROM bundle2bitstream LEFT JOIN item2bundle ON bundle2bitstream.bundle_id = item2bundle.bundle_id LEFT JOIN item ON item2bundle.item_id = item.uuid - WHERE item.withdrawn = 1 + WHERE item.withdrawn = true ); UPDATE resourcepolicy SET action_id = 12 where action_id = 0 and dspace_object in ( SELECT item2bundle.bundle_id FROM item2bundle LEFT JOIN item ON item2bundle.item_id = item.uuid - WHERE item.withdrawn = 1 + WHERE item.withdrawn = true ); \ No newline at end of file diff --git a/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V7.1_2021.10.18__Fix_MDV_place_after_migrating_from_DSpace_5.sql b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V7.1_2021.10.18__Fix_MDV_place_after_migrating_from_DSpace_5.sql index 3b649a321c..7506433cdd 100644 --- a/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V7.1_2021.10.18__Fix_MDV_place_after_migrating_from_DSpace_5.sql +++ b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V7.1_2021.10.18__Fix_MDV_place_after_migrating_from_DSpace_5.sql @@ -9,10 +9,11 @@ ---------------------------------------------------- -- Make sure the metadatavalue.place column starts at 0 instead of 1 ---------------------------------------------------- + CREATE LOCAL TEMPORARY TABLE mdv_minplace ( dspace_object_id UUID NOT NULL, metadata_field_id INT NOT NULL, - minplace INT NOT NULL, + minplace INT NOT NULL ); INSERT INTO mdv_minplace diff --git a/dspace-api/src/test/java/org/dspace/AbstractIntegrationTestWithDatabase.java b/dspace-api/src/test/java/org/dspace/AbstractIntegrationTestWithDatabase.java index cada4944e3..e27fb19a68 100644 --- a/dspace-api/src/test/java/org/dspace/AbstractIntegrationTestWithDatabase.java +++ b/dspace-api/src/test/java/org/dspace/AbstractIntegrationTestWithDatabase.java @@ -15,6 +15,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.dspace.app.launcher.ScriptLauncher; import org.dspace.app.scripts.handler.impl.TestDSpaceRunnableHandler; +import org.dspace.authority.AuthoritySearchService; import org.dspace.authority.MockAuthoritySolrServiceImpl; import org.dspace.authorize.AuthorizeException; import org.dspace.builder.AbstractBuilder; @@ -31,6 +32,7 @@ import org.dspace.kernel.ServiceManager; import org.dspace.services.factory.DSpaceServicesFactory; import org.dspace.statistics.MockSolrLoggerServiceImpl; import org.dspace.statistics.MockSolrStatisticsCore; +import org.dspace.statistics.SolrStatisticsCore; import org.dspace.storage.rdbms.DatabaseUtils; import org.jdom2.Document; import org.junit.After; @@ -183,15 +185,15 @@ public class AbstractIntegrationTestWithDatabase extends AbstractDSpaceIntegrati searchService.reset(); // Clear the statistics core. serviceManager - .getServiceByName(null, MockSolrStatisticsCore.class) + .getServiceByName(SolrStatisticsCore.class.getName(), MockSolrStatisticsCore.class) .reset(); MockSolrLoggerServiceImpl statisticsService = serviceManager - .getServiceByName(null, MockSolrLoggerServiceImpl.class); + .getServiceByName("solrLoggerService", MockSolrLoggerServiceImpl.class); statisticsService.reset(); MockAuthoritySolrServiceImpl authorityService = serviceManager - .getServiceByName(null, MockAuthoritySolrServiceImpl.class); + .getServiceByName(AuthoritySearchService.class.getName(), MockAuthoritySolrServiceImpl.class); authorityService.reset(); // Reload our ConfigurationService (to reset configs to defaults again) diff --git a/dspace-api/src/test/java/org/dspace/app/mediafilter/PoiWordFilterTest.java b/dspace-api/src/test/java/org/dspace/app/mediafilter/PoiWordFilterTest.java deleted file mode 100644 index 4d2353a29a..0000000000 --- a/dspace-api/src/test/java/org/dspace/app/mediafilter/PoiWordFilterTest.java +++ /dev/null @@ -1,181 +0,0 @@ -/** - * The contents of this file are subject to the license and copyright - * detailed in the LICENSE and NOTICE files at the root of the source - * tree and available online at - * - * http://www.dspace.org/license/ - */ -package org.dspace.app.mediafilter; - -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; - -import org.dspace.content.Item; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -/** - * Drive the POI-based MS Word filter. - * - * @author mwood - */ -public class PoiWordFilterTest { - - public PoiWordFilterTest() { - } - - @BeforeClass - public static void setUpClass() { - } - - @AfterClass - public static void tearDownClass() { - } - - @Before - public void setUp() { - } - - @After - public void tearDown() { - } - - /** - * Test of getFilteredName method, of class PoiWordFilter. - */ -/* - @Test - public void testGetFilteredName() - { - System.out.println("getFilteredName"); - String oldFilename = ""; - PoiWordFilter instance = new PoiWordFilter(); - String expResult = ""; - String result = instance.getFilteredName(oldFilename); - assertEquals(expResult, result); - // TODO review the generated test code and remove the default call to fail. - fail("The test case is a prototype."); - } -*/ - - /** - * Test of getBundleName method, of class PoiWordFilter. - */ -/* - @Test - public void testGetBundleName() - { - System.out.println("getBundleName"); - PoiWordFilter instance = new PoiWordFilter(); - String expResult = ""; - String result = instance.getBundleName(); - assertEquals(expResult, result); - // TODO review the generated test code and remove the default call to fail. - fail("The test case is a prototype."); - } -*/ - - /** - * Test of getFormatString method, of class PoiWordFilter. - */ -/* - @Test - public void testGetFormatString() - { - System.out.println("getFormatString"); - PoiWordFilter instance = new PoiWordFilter(); - String expResult = ""; - String result = instance.getFormatString(); - assertEquals(expResult, result); - // TODO review the generated test code and remove the default call to fail. - fail("The test case is a prototype."); - } -*/ - - /** - * Test of getDescription method, of class PoiWordFilter. - */ -/* - @Test - public void testGetDescription() - { - System.out.println("getDescription"); - PoiWordFilter instance = new PoiWordFilter(); - String expResult = ""; - String result = instance.getDescription(); - assertEquals(expResult, result); - // TODO review the generated test code and remove the default call to fail. - fail("The test case is a prototype."); - } -*/ - - /** - * Test of getDestinationStream method, of class PoiWordFilter. - * Read a constant .doc document and examine the extracted text. - * - * @throws java.lang.Exception passed through. - */ - @Test - public void testGetDestinationStreamDoc() - throws Exception { - System.out.println("getDestinationStream"); - Item currentItem = null; - InputStream source; - boolean verbose = false; - PoiWordFilter instance = new PoiWordFilter(); - InputStream result; - - source = getClass().getResourceAsStream("wordtest.doc"); - result = instance.getDestinationStream(currentItem, source, verbose); - assertTrue("Known content was not found", readAll(result).contains("quick brown fox")); - } - - /** - * Test of getDestinationStream method, of class PoiWordFilter. - * Read a constant .docx document and examine the extracted text. - * - * @throws java.lang.Exception passed through. - */ - @Test - public void testGetDestinationStreamDocx() - throws Exception { - System.out.println("getDestinationStream"); - Item currentItem = null; - InputStream source; - boolean verbose = false; - PoiWordFilter instance = new PoiWordFilter(); - InputStream result; - - source = getClass().getResourceAsStream("wordtest.docx"); - result = instance.getDestinationStream(currentItem, source, verbose); - assertTrue("Known content was not found", readAll(result).contains("quick brown fox")); - } - - /** - * Read the entire content of a stream into a String. - * - * @param stream a stream of UTF-8 characters. - * @return complete content of {@link stream} - * @throws IOException - */ - private static String readAll(InputStream stream) - throws IOException { - if (null == stream) { - return null; - } - - byte[] bytes = new byte[stream.available()]; - StringBuilder resultSb = new StringBuilder(bytes.length / 2); // Guess: average 2 bytes per character - int howmany; - while ((howmany = stream.read(bytes)) > 0) { - resultSb.append(new String(bytes, 0, howmany, StandardCharsets.UTF_8)); - } - return resultSb.toString(); - } -} diff --git a/dspace-api/src/test/java/org/dspace/app/mediafilter/TikaTextExtractionFilterTest.java b/dspace-api/src/test/java/org/dspace/app/mediafilter/TikaTextExtractionFilterTest.java new file mode 100644 index 0000000000..9db1ef7776 --- /dev/null +++ b/dspace-api/src/test/java/org/dspace/app/mediafilter/TikaTextExtractionFilterTest.java @@ -0,0 +1,323 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.mediafilter; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +import org.apache.commons.io.IOUtils; +import org.dspace.AbstractUnitTest; +import org.dspace.services.ConfigurationService; +import org.dspace.services.factory.DSpaceServicesFactory; +import org.junit.Test; + +/** + * Test the TikaTextExtractionFilter using test files for all major formats. + * The test files used below are all located at [dspace-api]/src/test/resources/org/dspace/app/mediafilter/ + * + * @author mwood + * @author Tim Donohue + */ +public class TikaTextExtractionFilterTest extends AbstractUnitTest { + + private ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService(); + + /** + * Test of getDestinationStream method using temp file for text extraction + * + * @throws java.lang.Exception passed through. + */ + @Test + public void testGetDestinationStreamWithUseTempFile() + throws Exception { + TikaTextExtractionFilter instance = new TikaTextExtractionFilter(); + + // Extract text from file with "use-temp-file=true" + configurationService.setProperty("textextractor.use-temp-file", "true"); + InputStream source = getClass().getResourceAsStream("test.pdf"); + InputStream result = instance.getDestinationStream(null, source, false); + String tempFileExtractedText = readAll(result); + + // Verify text extracted successfully + assertTrue("Known content was not found in .pdf", tempFileExtractedText.contains("quick brown fox")); + + // Now, extract text from same file using default, in-memory + configurationService.setProperty("textextractor.use-temp-file", "false"); + source = getClass().getResourceAsStream("test.pdf"); + result = instance.getDestinationStream(null, source, false); + String inMemoryExtractedText = readAll(result); + + // Verify the two results are equal + assertEquals("Extracted text via temp file is the same as in-memory.", + inMemoryExtractedText, tempFileExtractedText); + } + + /** + * Test of getDestinationStream method when max characters is less than file size + * + * @throws java.lang.Exception passed through. + */ + @Test + public void testGetDestinationStreamWithMaxChars() + throws Exception { + TikaTextExtractionFilter instance = new TikaTextExtractionFilter(); + + // Set "max-chars" to a small value of 100 chars, which is less than the text size of the file. + configurationService.setProperty("textextractor.max-chars", "100"); + InputStream source = getClass().getResourceAsStream("test.pdf"); + InputStream result = instance.getDestinationStream(null, source, false); + String extractedText = readAll(result); + + // Verify we have exactly the first 100 characters + assertEquals(100, extractedText.length()); + // Verify it has some text at the beginning of the file, but NOT text near the end + assertTrue("Known beginning content was found", extractedText.contains("This is a text.")); + assertFalse("Known ending content was not found", extractedText.contains("Emergency Broadcast System")); + } + + /** + * Test of getDestinationStream method using older Microsoft Word document. + * Read a constant .doc document and examine the extracted text. + * + * @throws java.lang.Exception passed through. + */ + @Test + public void testGetDestinationStreamWithDoc() + throws Exception { + TikaTextExtractionFilter instance = new TikaTextExtractionFilter(); + + InputStream source = getClass().getResourceAsStream("test.doc"); + InputStream result = instance.getDestinationStream(null, source, false); + assertTrue("Known content was not found in .doc", readAll(result).contains("quick brown fox")); + } + + /** + * Test of getDestinationStream method using newer Microsoft Word document. + * Read a constant .docx document and examine the extracted text. + * + * @throws java.lang.Exception passed through. + */ + @Test + public void testGetDestinationStreamWithDocx() + throws Exception { + TikaTextExtractionFilter instance = new TikaTextExtractionFilter(); + + InputStream source = getClass().getResourceAsStream("test.docx"); + InputStream result = instance.getDestinationStream(null, source, false); + assertTrue("Known content was not found in .docx", readAll(result).contains("quick brown fox")); + } + + /** + * Test of getDestinationStream method using an ODT document + * Read a constant .odt document and examine the extracted text. + * + * @throws java.lang.Exception passed through. + */ + @Test + public void testGetDestinationStreamWithODT() + throws Exception { + TikaTextExtractionFilter instance = new TikaTextExtractionFilter(); + + InputStream source = getClass().getResourceAsStream("test.odt"); + InputStream result = instance.getDestinationStream(null, source, false); + assertTrue("Known content was not found in .odt", readAll(result).contains("quick brown fox")); + } + + /** + * Test of getDestinationStream method using an RTF document + * Read a constant .rtf document and examine the extracted text. + * + * @throws java.lang.Exception passed through. + */ + @Test + public void testGetDestinationStreamWithRTF() + throws Exception { + TikaTextExtractionFilter instance = new TikaTextExtractionFilter(); + + InputStream source = getClass().getResourceAsStream("test.rtf"); + InputStream result = instance.getDestinationStream(null, source, false); + assertTrue("Known content was not found in .rtf", readAll(result).contains("quick brown fox")); + } + + /** + * Test of getDestinationStream method using a PDF document + * Read a constant .pdf document and examine the extracted text. + * + * @throws java.lang.Exception passed through. + */ + @Test + public void testGetDestinationStreamWithPDF() + throws Exception { + TikaTextExtractionFilter instance = new TikaTextExtractionFilter(); + + InputStream source = getClass().getResourceAsStream("test.pdf"); + InputStream result = instance.getDestinationStream(null, source, false); + assertTrue("Known content was not found in .pdf", readAll(result).contains("quick brown fox")); + } + + /** + * Test of getDestinationStream method using an HTML document + * Read a constant .html document and examine the extracted text. + * + * @throws java.lang.Exception passed through. + */ + @Test + public void testGetDestinationStreamWithHTML() + throws Exception { + TikaTextExtractionFilter instance = new TikaTextExtractionFilter(); + + InputStream source = getClass().getResourceAsStream("test.html"); + InputStream result = instance.getDestinationStream(null, source, false); + assertTrue("Known content was not found in .html", readAll(result).contains("quick brown fox")); + } + + /** + * Test of getDestinationStream method using a TXT document + * Read a constant .txt document and examine the extracted text. + * + * @throws java.lang.Exception passed through. + */ + @Test + public void testGetDestinationStreamWithTxt() + throws Exception { + TikaTextExtractionFilter instance = new TikaTextExtractionFilter(); + + InputStream source = getClass().getResourceAsStream("test.txt"); + InputStream result = instance.getDestinationStream(null, source, false); + assertTrue("Known content was not found in .txt", readAll(result).contains("quick brown fox")); + } + + /** + * Test of getDestinationStream method using a CSV document + * Read a constant .csv document and examine the extracted text. + * + * @throws java.lang.Exception passed through. + */ + @Test + public void testGetDestinationStreamWithCsv() + throws Exception { + TikaTextExtractionFilter instance = new TikaTextExtractionFilter(); + + InputStream source = getClass().getResourceAsStream("test.csv"); + InputStream result = instance.getDestinationStream(null, source, false); + assertTrue("Known content was not found in .csv", readAll(result).contains("data3,3")); + } + + /** + * Test of getDestinationStream method using an XLS document + * Read a constant .xls document and examine the extracted text. + * + * @throws java.lang.Exception passed through. + */ + @Test + public void testGetDestinationStreamWithXLS() + throws Exception { + TikaTextExtractionFilter instance = new TikaTextExtractionFilter(); + + InputStream source = getClass().getResourceAsStream("test.xls"); + InputStream result = instance.getDestinationStream(null, source, false); + assertTrue("Known content was not found in .xls", readAll(result).contains("data3,3")); + } + + /** + * Test of getDestinationStream method using an XLSX document + * Read a constant .xlsx document and examine the extracted text. + * + * @throws java.lang.Exception passed through. + */ + @Test + public void testGetDestinationStreamWithXLSX() + throws Exception { + TikaTextExtractionFilter instance = new TikaTextExtractionFilter(); + + InputStream source = getClass().getResourceAsStream("test.xlsx"); + InputStream result = instance.getDestinationStream(null, source, false); + assertTrue("Known content was not found in .xlsx", readAll(result).contains("data3,3")); + } + + /** + * Test of getDestinationStream method using an ODS document + * Read a constant .ods document and examine the extracted text. + * + * @throws java.lang.Exception passed through. + */ + @Test + public void testGetDestinationStreamWithODS() + throws Exception { + TikaTextExtractionFilter instance = new TikaTextExtractionFilter(); + + InputStream source = getClass().getResourceAsStream("test.ods"); + InputStream result = instance.getDestinationStream(null, source, false); + assertTrue("Known content was not found in .ods", readAll(result).contains("Data on the second sheet")); + } + + /** + * Test of getDestinationStream method using an PPT document + * Read a constant .ppt document and examine the extracted text. + * + * @throws java.lang.Exception passed through. + */ + @Test + public void testGetDestinationStreamWithPPT() + throws Exception { + TikaTextExtractionFilter instance = new TikaTextExtractionFilter(); + + InputStream source = getClass().getResourceAsStream("test.ppt"); + InputStream result = instance.getDestinationStream(null, source, false); + assertTrue("Known content was not found in .ppt", readAll(result).contains("quick brown fox")); + } + + /** + * Test of getDestinationStream method using an PPTX document + * Read a constant .pptx document and examine the extracted text. + * + * @throws java.lang.Exception passed through. + */ + @Test + public void testGetDestinationStreamWithPPTX() + throws Exception { + TikaTextExtractionFilter instance = new TikaTextExtractionFilter(); + + InputStream source = getClass().getResourceAsStream("test.pptx"); + InputStream result = instance.getDestinationStream(null, source, false); + assertTrue("Known content was not found in .pptx", readAll(result).contains("quick brown fox")); + } + + /** + * Test of getDestinationStream method using an ODP document + * Read a constant .odp document and examine the extracted text. + * + * @throws java.lang.Exception passed through. + */ + @Test + public void testGetDestinationStreamWithODP() + throws Exception { + TikaTextExtractionFilter instance = new TikaTextExtractionFilter(); + + InputStream source = getClass().getResourceAsStream("test.odp"); + InputStream result = instance.getDestinationStream(null, source, false); + assertTrue("Known content was not found in .odp", readAll(result).contains("quick brown fox")); + } + + /** + * Read the entire content of a stream into a String. + * + * @param stream a stream of UTF-8 characters. + * @return complete content of stream as a String + * @throws IOException + */ + private static String readAll(InputStream stream) + throws IOException { + return IOUtils.toString(stream, StandardCharsets.UTF_8); + } +} diff --git a/dspace-api/src/test/java/org/dspace/app/util/ConfigurationIT.java b/dspace-api/src/test/java/org/dspace/app/util/ConfigurationIT.java new file mode 100644 index 0000000000..388b467e97 --- /dev/null +++ b/dspace-api/src/test/java/org/dspace/app/util/ConfigurationIT.java @@ -0,0 +1,268 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ + +package org.dspace.app.util; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.arrayWithSize; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.collection.IsArrayContainingInAnyOrder.arrayContainingInAnyOrder; +import static org.junit.Assert.assertEquals; + +import org.dspace.AbstractDSpaceTest; +import org.dspace.services.ConfigurationService; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.contrib.java.lang.system.Assertion; +import org.junit.contrib.java.lang.system.ExpectedSystemExit; +import org.junit.contrib.java.lang.system.SystemErrRule; +import org.junit.contrib.java.lang.system.SystemOutRule; + +/** + * Tests for configuration utilities. + * + * Because our command-line tools call System.exit(), we can't expect any code + * (such as assertions) following the call to main() to be executed. Instead we + * set up expectations in advance and attach them to an exit() trapper. + * + * @author mhwood + */ +public class ConfigurationIT + extends AbstractDSpaceTest { + + private static ConfigurationService cfg; + + private static final String SINGLE_PROPERTY = "test.single"; + private static final String SINGLE_VALUE = "value"; + + private static final String ARRAY_PROPERTY = "test.array"; + private static final String[] ARRAY_VALUE = { "one", "two" }; + + private static final String PLACEHOLDER_PROPERTY = "test.substituted"; + private static final String PLACEHOLDER_VALUE = "insert ${test.single} here"; // Keep aligned with SINGLE_NAME + private static final String SUBSTITUTED_VALUE = "insert value here"; // Keep aligned with SINGLE_VALUE + + private static final String MISSING_PROPERTY = "test.missing"; + + /** Capture standard output. */ + @Rule + public final SystemOutRule systemOutRule = new SystemOutRule(); + + /** Capture standard error. */ + @Rule + public final SystemErrRule systemErrRule = new SystemErrRule(); + + /** Capture System.exit() value. */ + @Rule + public final ExpectedSystemExit expectedSystemExit = ExpectedSystemExit.none(); + + /** + * Create some expected properties before all tests. + */ + @BeforeClass + public static void setupSuite() { + cfg = kernelImpl.getConfigurationService(); + + cfg.setProperty(SINGLE_PROPERTY, SINGLE_VALUE); + cfg.setProperty(ARRAY_PROPERTY, ARRAY_VALUE); + cfg.setProperty(PLACEHOLDER_PROPERTY, PLACEHOLDER_VALUE); + cfg.setProperty(MISSING_PROPERTY, null); // Ensure that this one is undefined + } + + /** + * After all tests, remove the properties that were created at entry. + */ + @AfterClass + public static void teardownSuite() { + if (null != cfg) { + cfg.setProperty(SINGLE_PROPERTY, null); + cfg.setProperty(ARRAY_PROPERTY, null); + cfg.setProperty(PLACEHOLDER_PROPERTY, null); + } + } + + /** + * Test fetching all values of a single-valued property. + */ + @Test + public void testMainAllSingle() { + String[] argv; + argv = new String[] { + "--property", SINGLE_PROPERTY + }; + expectedSystemExit.expectSystemExitWithStatus(0); + expectedSystemExit.checkAssertionAfterwards(new Assertion() { + @Override public void checkAssertion() { + String[] output = systemOutRule.getLogWithNormalizedLineSeparator() + .split("\n"); + assertThat(output, arrayWithSize(1)); + } + }); + expectedSystemExit.checkAssertionAfterwards(new Assertion() { + @Override public void checkAssertion() { + String[] output = systemOutRule.getLogWithNormalizedLineSeparator() + .split("\n"); + assertThat(output[0], equalTo(SINGLE_VALUE)); + } + }); + systemOutRule.enableLog(); + Configuration.main(argv); + } + + /** + * Test fetching all values of an array property. + */ + @Test + public void testMainAllArray() { + String[] argv; + argv = new String[] { + "--property", ARRAY_PROPERTY + }; + expectedSystemExit.expectSystemExitWithStatus(0); + expectedSystemExit.checkAssertionAfterwards(new Assertion() { + @Override public void checkAssertion() { + String[] output = systemOutRule.getLogWithNormalizedLineSeparator() + .split("\n"); + assertThat(output, arrayWithSize(ARRAY_VALUE.length)); + } + }); + expectedSystemExit.checkAssertionAfterwards(new Assertion() { + @Override public void checkAssertion() { + String[] output = systemOutRule.getLogWithNormalizedLineSeparator() + .split("\n"); + assertThat(output, arrayContainingInAnyOrder(ARRAY_VALUE)); + } + }); + systemOutRule.enableLog(); + Configuration.main(argv); + } + + /** + * Test fetching all values of a single-valued property containing property + * placeholders. + */ + @Test + public void testMainAllSubstitution() { + String[] argv; + argv = new String[] { + "--property", PLACEHOLDER_PROPERTY + }; + expectedSystemExit.expectSystemExitWithStatus(0); + expectedSystemExit.checkAssertionAfterwards(new Assertion() { + @Override public void checkAssertion() { + String[] output = systemOutRule.getLogWithNormalizedLineSeparator() + .split("\n"); + assertThat(output, arrayWithSize(1)); + } + }); + expectedSystemExit.checkAssertionAfterwards(new Assertion() { + @Override public void checkAssertion() { + String[] output = systemOutRule.getLogWithNormalizedLineSeparator() + .split("\n"); + assertThat(output[0], equalTo(SUBSTITUTED_VALUE)); + } + }); + systemOutRule.enableLog(); + Configuration.main(argv); + } + + /** + * Test fetching all values of a single-valued property containing property + * placeholders, suppressing property substitution. + */ + @Test + public void testMainAllRaw() { + // Can it handle a raw property (with substitution placeholders)? + String[] argv; + argv = new String[] { + "--property", PLACEHOLDER_PROPERTY, + "--raw" + }; + expectedSystemExit.expectSystemExitWithStatus(0); + expectedSystemExit.checkAssertionAfterwards(new Assertion() { + @Override public void checkAssertion() { + String[] output = systemOutRule.getLogWithNormalizedLineSeparator() + .split("\n"); + assertThat(output, arrayWithSize(1)); + } + }); + expectedSystemExit.checkAssertionAfterwards(new Assertion() { + @Override public void checkAssertion() { + String[] output = systemOutRule.getLogWithNormalizedLineSeparator() + .split("\n"); + assertThat(output[0], equalTo(PLACEHOLDER_VALUE)); + } + }); + systemOutRule.enableLog(); + Configuration.main(argv); + } + + /** + * Test fetching all values of an undefined property. + */ + @Test + public void testMainAllUndefined() { + // Can it handle an undefined property? + String[] argv; + argv = new String[] { + "--property", MISSING_PROPERTY + }; + expectedSystemExit.expectSystemExitWithStatus(0); + expectedSystemExit.checkAssertionAfterwards(new Assertion() { + @Override public void checkAssertion() { + String outputs = systemOutRule.getLogWithNormalizedLineSeparator(); + String[] output = outputs.split("\n"); + assertThat(output, arrayWithSize(0)); // Huh? Shouldn't split() return { "" } ? + } + }); + systemOutRule.enableLog(); + Configuration.main(argv); + } + + /** + * Test fetching only the first value of an array property. + */ + @Test + public void testMainFirstArray() { + String[] argv = new String[] { + "--property", ARRAY_PROPERTY, + "--first" + }; + expectedSystemExit.expectSystemExitWithStatus(0); + expectedSystemExit.checkAssertionAfterwards(() -> { + String outputs = systemOutRule.getLogWithNormalizedLineSeparator(); + String[] output = outputs.split("\n"); + assertThat(output, arrayWithSize(1)); + assertEquals("--first should return first value", output[0], ARRAY_VALUE[0]); + }); + systemOutRule.enableLog(); + Configuration.main(argv); + } + + /** + * Test fetching a single-valued property using {@code --first} + */ + @Test + public void testMainFirstSingle() { + String[] argv = new String[] { + "--property", SINGLE_PROPERTY, + "--first" + }; + expectedSystemExit.expectSystemExitWithStatus(0); + expectedSystemExit.checkAssertionAfterwards(() -> { + String outputs = systemOutRule.getLogWithNormalizedLineSeparator(); + String[] output = outputs.split("\n"); + assertThat(output, arrayWithSize(1)); + assertEquals("--first should return only value", output[0], SINGLE_VALUE); + }); + systemOutRule.enableLog(); + Configuration.main(argv); + } +} diff --git a/dspace-api/src/test/java/org/dspace/builder/BitstreamBuilder.java b/dspace-api/src/test/java/org/dspace/builder/BitstreamBuilder.java index 283091778e..424833e5cc 100644 --- a/dspace-api/src/test/java/org/dspace/builder/BitstreamBuilder.java +++ b/dspace-api/src/test/java/org/dspace/builder/BitstreamBuilder.java @@ -18,6 +18,7 @@ import org.dspace.content.BitstreamFormat; import org.dspace.content.Bundle; import org.dspace.content.Item; import org.dspace.content.service.DSpaceObjectService; +import org.dspace.core.Constants; import org.dspace.core.Context; import org.dspace.eperson.Group; @@ -26,8 +27,6 @@ import org.dspace.eperson.Group; */ public class BitstreamBuilder extends AbstractDSpaceObjectBuilder { - public static final String ORIGINAL = "ORIGINAL"; - private Bitstream bitstream; private Item item; private Group readerGroup; @@ -158,12 +157,12 @@ public class BitstreamBuilder extends AbstractDSpaceObjectBuilder { } private Bundle getOriginalBundle(Item item) throws SQLException, AuthorizeException { - List bundles = itemService.getBundles(item, ORIGINAL); + List bundles = itemService.getBundles(item, Constants.CONTENT_BUNDLE_NAME); Bundle targetBundle = null; if (bundles.size() < 1) { // not found, create a new one - targetBundle = bundleService.create(context, item, ORIGINAL); + targetBundle = bundleService.create(context, item, Constants.CONTENT_BUNDLE_NAME); } else { // put bitstreams into first bundle targetBundle = bundles.iterator().next(); diff --git a/dspace-api/src/test/java/org/dspace/iiif/canvasdimension/CanvasDimensionsIT.java b/dspace-api/src/test/java/org/dspace/iiif/canvasdimension/CanvasDimensionsIT.java index 502266da06..2711f46c31 100644 --- a/dspace-api/src/test/java/org/dspace/iiif/canvasdimension/CanvasDimensionsIT.java +++ b/dspace-api/src/test/java/org/dspace/iiif/canvasdimension/CanvasDimensionsIT.java @@ -15,6 +15,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; +import org.apache.commons.lang.StringUtils; import org.dspace.AbstractIntegrationTestWithDatabase; import org.dspace.builder.BitstreamBuilder; import org.dspace.builder.CollectionBuilder; @@ -408,7 +409,7 @@ public class CanvasDimensionsIT extends AbstractIntegrationTestWithDatabase { execCanvasScriptWithMaxRecs(id); // check System.out for number of items processed. - assertEquals("2 IIIF items were processed.\n", outContent.toString()); + assertEquals("2 IIIF items were processed.", StringUtils.chomp(outContent.toString())); } @Test diff --git a/dspace-api/src/test/java/org/dspace/solr/MockSolrServer.java b/dspace-api/src/test/java/org/dspace/solr/MockSolrServer.java index e80d5f8e17..aed0c088c3 100644 --- a/dspace-api/src/test/java/org/dspace/solr/MockSolrServer.java +++ b/dspace-api/src/test/java/org/dspace/solr/MockSolrServer.java @@ -171,6 +171,7 @@ public class MockSolrServer { * Discard the embedded Solr container. */ private static synchronized void destroyContainer() { + container.shutdown(); container = null; log.info("SOLR CoreContainer destroyed"); } diff --git a/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.csv b/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.csv new file mode 100644 index 0000000000..07c22ff0bf --- /dev/null +++ b/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.csv @@ -0,0 +1,4 @@ +row1,row2,row3,row4 +"data1,2","data 2,2","data3,2","data4,2" +"data1,3","data 2,3","data3,3","data4,3" +"data1,4","data2,4","data3,4","data4,4" diff --git a/dspace-api/src/test/resources/org/dspace/app/mediafilter/wordtest.doc b/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.doc similarity index 100% rename from dspace-api/src/test/resources/org/dspace/app/mediafilter/wordtest.doc rename to dspace-api/src/test/resources/org/dspace/app/mediafilter/test.doc diff --git a/dspace-api/src/test/resources/org/dspace/app/mediafilter/wordtest.docx b/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.docx similarity index 100% rename from dspace-api/src/test/resources/org/dspace/app/mediafilter/wordtest.docx rename to dspace-api/src/test/resources/org/dspace/app/mediafilter/test.docx diff --git a/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.html b/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.html new file mode 100644 index 0000000000..7655f566cc --- /dev/null +++ b/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.html @@ -0,0 +1,53 @@ + + + + +A Text Extraction Test Document for DSpace + + + + +
    + +

    A Text Extraction Test Document

    + +

    for

    + +

    DSpace

    + +

    + +

    This is a text. For the next sixty seconds this software +will conduct a test of the DSpace text extraction facility. This is only a +text.

    + +

    This is a paragraph that followed the first that lived in +the document that Jack built.

    + +

    Lorem ipsum dolor sit amet. The quick brown fox jumped over +the lazy dog. Yow! Are we having fun yet?

    + +

    This has been a test of the DSpace text extraction system. +In the event of actual content you would care what is written here.

    + +
    + +
    + +
    + +
    + +

    Tip o’ the hat to the U.S. Emergency Broadcast System for the format that I have +irreverently borrowed.

    + +
    + + + + + + diff --git a/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.odp b/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.odp new file mode 100644 index 0000000000..4701884a8a Binary files /dev/null and b/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.odp differ diff --git a/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.ods b/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.ods new file mode 100644 index 0000000000..94ad873c1a Binary files /dev/null and b/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.ods differ diff --git a/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.odt b/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.odt new file mode 100644 index 0000000000..3c996a1f46 Binary files /dev/null and b/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.odt differ diff --git a/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.pdf b/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.pdf new file mode 100644 index 0000000000..5b3749cbff Binary files /dev/null and b/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.pdf differ diff --git a/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.ppt b/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.ppt new file mode 100644 index 0000000000..bb3a3d6b41 Binary files /dev/null and b/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.ppt differ diff --git a/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.pptx b/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.pptx new file mode 100644 index 0000000000..2c27ad1630 Binary files /dev/null and b/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.pptx differ diff --git a/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.rtf b/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.rtf new file mode 100644 index 0000000000..3b841917b2 --- /dev/null +++ b/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.rtf @@ -0,0 +1,239 @@ +{\rtf1\adeflang1025\ansi\ansicpg1252\uc1\adeff46\deff0\stshfdbch45\stshfloch43\stshfhich43\stshfbi46\deflang1033\deflangfe1033\themelang1033\themelangfe0\themelangcs0{\fonttbl{\f34\fbidi \froman\fcharset0\fprq2{\*\panose 02040503050406030204}Cambria Math;}{\f43\fbidi \froman\fcharset0\fprq2 Liberation Serif{\*\falt Times New Roman};} +{\f44\fbidi \fswiss\fcharset0\fprq2 Liberation Sans{\*\falt Arial};}{\f45\fbidi \froman\fcharset0\fprq0{\*\panose 00000000000000000000}AR PL SungtiL GB;}{\f46\fbidi \froman\fcharset0\fprq0{\*\panose 00000000000000000000}Lohit Hindi;} +{\flomajor\f31500\fbidi \froman\fcharset0\fprq2{\*\panose 02020603050405020304}Times New Roman;}{\fdbmajor\f31501\fbidi \froman\fcharset0\fprq2{\*\panose 02020603050405020304}Times New Roman;} +{\fhimajor\f31502\fbidi \fswiss\fcharset0\fprq2{\*\panose 020f0302020204030204}Calibri Light;}{\fbimajor\f31503\fbidi \froman\fcharset0\fprq2{\*\panose 02020603050405020304}Times New Roman;} +{\flominor\f31504\fbidi \froman\fcharset0\fprq2{\*\panose 02020603050405020304}Times New Roman;}{\fdbminor\f31505\fbidi \froman\fcharset0\fprq2{\*\panose 02020603050405020304}Times New Roman;} +{\fhiminor\f31506\fbidi \fswiss\fcharset0\fprq2{\*\panose 020f0502020204030204}Calibri;}{\fbiminor\f31507\fbidi \froman\fcharset0\fprq2{\*\panose 02020603050405020304}Times New Roman;}{\f1504\fbidi \froman\fcharset238\fprq2 Cambria Math CE;} +{\f1505\fbidi \froman\fcharset204\fprq2 Cambria Math Cyr;}{\f1507\fbidi \froman\fcharset161\fprq2 Cambria Math Greek;}{\f1508\fbidi \froman\fcharset162\fprq2 Cambria Math Tur;}{\f1511\fbidi \froman\fcharset186\fprq2 Cambria Math Baltic;} +{\f1512\fbidi \froman\fcharset163\fprq2 Cambria Math (Vietnamese);}{\flomajor\f31508\fbidi \froman\fcharset238\fprq2 Times New Roman CE;}{\flomajor\f31509\fbidi \froman\fcharset204\fprq2 Times New Roman Cyr;} +{\flomajor\f31511\fbidi \froman\fcharset161\fprq2 Times New Roman Greek;}{\flomajor\f31512\fbidi \froman\fcharset162\fprq2 Times New Roman Tur;}{\flomajor\f31513\fbidi \froman\fcharset177\fprq2 Times New Roman (Hebrew);} +{\flomajor\f31514\fbidi \froman\fcharset178\fprq2 Times New Roman (Arabic);}{\flomajor\f31515\fbidi \froman\fcharset186\fprq2 Times New Roman Baltic;}{\flomajor\f31516\fbidi \froman\fcharset163\fprq2 Times New Roman (Vietnamese);} +{\fdbmajor\f31518\fbidi \froman\fcharset238\fprq2 Times New Roman CE;}{\fdbmajor\f31519\fbidi \froman\fcharset204\fprq2 Times New Roman Cyr;}{\fdbmajor\f31521\fbidi \froman\fcharset161\fprq2 Times New Roman Greek;} +{\fdbmajor\f31522\fbidi \froman\fcharset162\fprq2 Times New Roman Tur;}{\fdbmajor\f31523\fbidi \froman\fcharset177\fprq2 Times New Roman (Hebrew);}{\fdbmajor\f31524\fbidi \froman\fcharset178\fprq2 Times New Roman (Arabic);} +{\fdbmajor\f31525\fbidi \froman\fcharset186\fprq2 Times New Roman Baltic;}{\fdbmajor\f31526\fbidi \froman\fcharset163\fprq2 Times New Roman (Vietnamese);}{\fhimajor\f31528\fbidi \fswiss\fcharset238\fprq2 Calibri Light CE;} +{\fhimajor\f31529\fbidi \fswiss\fcharset204\fprq2 Calibri Light Cyr;}{\fhimajor\f31531\fbidi \fswiss\fcharset161\fprq2 Calibri Light Greek;}{\fhimajor\f31532\fbidi \fswiss\fcharset162\fprq2 Calibri Light Tur;} +{\fhimajor\f31533\fbidi \fswiss\fcharset177\fprq2 Calibri Light (Hebrew);}{\fhimajor\f31534\fbidi \fswiss\fcharset178\fprq2 Calibri Light (Arabic);}{\fhimajor\f31535\fbidi \fswiss\fcharset186\fprq2 Calibri Light Baltic;} +{\fhimajor\f31536\fbidi \fswiss\fcharset163\fprq2 Calibri Light (Vietnamese);}{\fbimajor\f31538\fbidi \froman\fcharset238\fprq2 Times New Roman CE;}{\fbimajor\f31539\fbidi \froman\fcharset204\fprq2 Times New Roman Cyr;} +{\fbimajor\f31541\fbidi \froman\fcharset161\fprq2 Times New Roman Greek;}{\fbimajor\f31542\fbidi \froman\fcharset162\fprq2 Times New Roman Tur;}{\fbimajor\f31543\fbidi \froman\fcharset177\fprq2 Times New Roman (Hebrew);} +{\fbimajor\f31544\fbidi \froman\fcharset178\fprq2 Times New Roman (Arabic);}{\fbimajor\f31545\fbidi \froman\fcharset186\fprq2 Times New Roman Baltic;}{\fbimajor\f31546\fbidi \froman\fcharset163\fprq2 Times New Roman (Vietnamese);} +{\flominor\f31548\fbidi \froman\fcharset238\fprq2 Times New Roman CE;}{\flominor\f31549\fbidi \froman\fcharset204\fprq2 Times New Roman Cyr;}{\flominor\f31551\fbidi \froman\fcharset161\fprq2 Times New Roman Greek;} +{\flominor\f31552\fbidi \froman\fcharset162\fprq2 Times New Roman Tur;}{\flominor\f31553\fbidi \froman\fcharset177\fprq2 Times New Roman (Hebrew);}{\flominor\f31554\fbidi \froman\fcharset178\fprq2 Times New Roman (Arabic);} +{\flominor\f31555\fbidi \froman\fcharset186\fprq2 Times New Roman Baltic;}{\flominor\f31556\fbidi \froman\fcharset163\fprq2 Times New Roman (Vietnamese);}{\fdbminor\f31558\fbidi \froman\fcharset238\fprq2 Times New Roman CE;} +{\fdbminor\f31559\fbidi \froman\fcharset204\fprq2 Times New Roman Cyr;}{\fdbminor\f31561\fbidi \froman\fcharset161\fprq2 Times New Roman Greek;}{\fdbminor\f31562\fbidi \froman\fcharset162\fprq2 Times New Roman Tur;} +{\fdbminor\f31563\fbidi \froman\fcharset177\fprq2 Times New Roman (Hebrew);}{\fdbminor\f31564\fbidi \froman\fcharset178\fprq2 Times New Roman (Arabic);}{\fdbminor\f31565\fbidi \froman\fcharset186\fprq2 Times New Roman Baltic;} +{\fdbminor\f31566\fbidi \froman\fcharset163\fprq2 Times New Roman (Vietnamese);}{\fhiminor\f31568\fbidi \fswiss\fcharset238\fprq2 Calibri CE;}{\fhiminor\f31569\fbidi \fswiss\fcharset204\fprq2 Calibri Cyr;} +{\fhiminor\f31571\fbidi \fswiss\fcharset161\fprq2 Calibri Greek;}{\fhiminor\f31572\fbidi \fswiss\fcharset162\fprq2 Calibri Tur;}{\fhiminor\f31573\fbidi \fswiss\fcharset177\fprq2 Calibri (Hebrew);} +{\fhiminor\f31574\fbidi \fswiss\fcharset178\fprq2 Calibri (Arabic);}{\fhiminor\f31575\fbidi \fswiss\fcharset186\fprq2 Calibri Baltic;}{\fhiminor\f31576\fbidi \fswiss\fcharset163\fprq2 Calibri (Vietnamese);} +{\fbiminor\f31578\fbidi \froman\fcharset238\fprq2 Times New Roman CE;}{\fbiminor\f31579\fbidi \froman\fcharset204\fprq2 Times New Roman Cyr;}{\fbiminor\f31581\fbidi \froman\fcharset161\fprq2 Times New Roman Greek;} +{\fbiminor\f31582\fbidi \froman\fcharset162\fprq2 Times New Roman Tur;}{\fbiminor\f31583\fbidi \froman\fcharset177\fprq2 Times New Roman (Hebrew);}{\fbiminor\f31584\fbidi \froman\fcharset178\fprq2 Times New Roman (Arabic);} +{\fbiminor\f31585\fbidi \froman\fcharset186\fprq2 Times New Roman Baltic;}{\fbiminor\f31586\fbidi \froman\fcharset163\fprq2 Times New Roman (Vietnamese);}{\f1164\fbidi \froman\fcharset238\fprq2 Times New Roman CE;} +{\f1165\fbidi \froman\fcharset204\fprq2 Times New Roman Cyr;}{\f1167\fbidi \froman\fcharset161\fprq2 Times New Roman Greek;}{\f1168\fbidi \froman\fcharset162\fprq2 Times New Roman Tur;}{\f1169\fbidi \froman\fcharset177\fprq2 Times New Roman (Hebrew);} +{\f1170\fbidi \froman\fcharset178\fprq2 Times New Roman (Arabic);}{\f1171\fbidi \froman\fcharset186\fprq2 Times New Roman Baltic;}{\f1172\fbidi \froman\fcharset163\fprq2 Times New Roman (Vietnamese);}}{\colortbl;\red0\green0\blue0;\red0\green0\blue255; +\red0\green255\blue255;\red0\green255\blue0;\red255\green0\blue255;\red255\green0\blue0;\red255\green255\blue0;\red255\green255\blue255;\red0\green0\blue128;\red0\green128\blue128;\red0\green128\blue0;\red128\green0\blue128;\red128\green0\blue0; +\red128\green128\blue0;\red128\green128\blue128;\red192\green192\blue192;\red0\green0\blue0;\red0\green0\blue0;}{\*\defchp \fs24\lang1033\langfe2052\loch\af43\hich\af43\dbch\af45\langfenp2052 }{\*\defpap +\ql \li0\ri0\widctlpar\wrapdefault\aspalpha\aspnum\faauto\adjustright\rin0\lin0\itap0 }\noqfpromote {\stylesheet{\ql \li0\ri0\widctlpar\wrapdefault\aspalpha\aspnum\faauto\adjustright\rin0\lin0\itap0 \rtlch\fcs1 \af46\afs24\alang1081 \ltrch\fcs0 +\fs24\lang1033\langfe2052\loch\f43\hich\af43\dbch\af45\cgrid\langnp1033\langfenp2052 \snext0 \sqformat \spriority0 Normal;}{\*\cs10 \additive \ssemihidden \sunhideused \spriority1 Default Paragraph Font;}{\* +\ts11\tsrowd\trftsWidthB3\trpaddl108\trpaddr108\trpaddfl3\trpaddft3\trpaddfb3\trpaddfr3\trcbpat1\trcfpat1\tblind0\tblindtype3\tsvertalt\tsbrdrt\tsbrdrl\tsbrdrb\tsbrdrr\tsbrdrdgl\tsbrdrdgr\tsbrdrh\tsbrdrv +\ql \li0\ri0\widctlpar\wrapdefault\aspalpha\aspnum\faauto\adjustright\rin0\lin0\itap0 \rtlch\fcs1 \af46\afs24\alang1081 \ltrch\fcs0 \fs24\lang1033\langfe2052\loch\f43\hich\af43\dbch\af45\cgrid\langnp1033\langfenp2052 \snext11 \ssemihidden \sunhideused +Normal Table;}{\*\cs15 \additive \sqformat \spriority0 Footnote Characters;}{\*\cs16 \additive \super \spriority0 Footnote Anchor;}{\*\cs17 \additive \super \spriority0 Endnote Anchor;}{\*\cs18 \additive \sqformat \spriority0 Endnote Characters;}{ +\s19\ql \li0\ri0\sb240\sa120\keepn\widctlpar\wrapdefault\aspalpha\aspnum\faauto\adjustright\rin0\lin0\itap0 \rtlch\fcs1 \af46\afs28\alang1081 \ltrch\fcs0 \fs28\lang1033\langfe2052\loch\f44\hich\af44\dbch\af45\cgrid\langnp1033\langfenp2052 +\sbasedon0 \snext20 \sqformat \spriority0 Heading;}{\s20\ql \li0\ri0\sa140\sl288\slmult1\widctlpar\wrapdefault\aspalpha\aspnum\faauto\adjustright\rin0\lin0\itap0 \rtlch\fcs1 \af46\afs24\alang1081 \ltrch\fcs0 +\fs24\lang1033\langfe2052\loch\f43\hich\af43\dbch\af45\cgrid\langnp1033\langfenp2052 \sbasedon0 \snext20 \spriority0 Body Text;}{\s21\ql \li0\ri0\sa140\sl288\slmult1\widctlpar\wrapdefault\aspalpha\aspnum\faauto\adjustright\rin0\lin0\itap0 \rtlch\fcs1 +\af46\afs24\alang1081 \ltrch\fcs0 \fs24\lang1033\langfe2052\loch\f43\hich\af43\dbch\af45\cgrid\langnp1033\langfenp2052 \sbasedon20 \snext21 \spriority0 List;}{ +\s22\ql \li0\ri0\sb120\sa120\widctlpar\noline\wrapdefault\aspalpha\aspnum\faauto\adjustright\rin0\lin0\itap0 \rtlch\fcs1 \ai\af46\afs24\alang1081 \ltrch\fcs0 \i\fs24\lang1033\langfe2052\loch\f43\hich\af43\dbch\af45\cgrid\langnp1033\langfenp2052 +\sbasedon0 \snext22 \sqformat \spriority0 caption;}{\s23\ql \li0\ri0\widctlpar\noline\wrapdefault\aspalpha\aspnum\faauto\adjustright\rin0\lin0\itap0 \rtlch\fcs1 \af46\afs24\alang1081 \ltrch\fcs0 +\fs24\lang1033\langfe2052\loch\f43\hich\af43\dbch\af45\cgrid\langnp1033\langfenp2052 \sbasedon0 \snext23 \sqformat \spriority0 Index;}{\s24\ql \fi-339\li339\ri0\widctlpar\noline\wrapdefault\aspalpha\aspnum\faauto\adjustright\rin0\lin339\itap0 \rtlch\fcs1 +\af46\afs20\alang1081 \ltrch\fcs0 \fs20\lang1033\langfe2052\loch\f43\hich\af43\dbch\af45\cgrid\langnp1033\langfenp2052 \sbasedon0 \snext24 \spriority0 footnote text;}}{\*\rsidtbl \rsid6097384\rsid16590483\rsid16671749}{\mmathPr\mmathFont34\mbrkBin0 +\mbrkBinSub0\msmallFrac0\mdispDef1\mlMargin0\mrMargin0\mdefJc1\mwrapIndent1440\mintLim0\mnaryLim1}{\info{\title A Text Extraction Test Document for DSpace}{\author Mark Wood}{\operator Tim Donohue}{\creatim\yr2022\mo3\dy30\hr13\min54} +{\revtim\yr2022\mo3\dy30\hr13\min54}{\version2}{\edmins0}{\nofpages1}{\nofwords75}{\nofchars433}{\nofcharsws507}{\vern43}}{\*\xmlnstbl {\xmlns1 http://schemas.microsoft.com/office/word/2003/wordml}} +\paperw12240\paperh15840\margl1134\margr1134\margt1134\margb1134\gutter0\ltrsect +\deftab709\widowctrl\ftnbj\aenddoc\trackmoves0\trackformatting1\donotembedsysfont1\relyonvml0\donotembedlingdata0\grfdocevents0\validatexml1\showplaceholdtext0\ignoremixedcontent0\saveinvalidxml0\showxmlerrors1 +\noxlattoyen\expshrtn\noultrlspc\dntblnsbdb\nospaceforul\formshade\horzdoc\dgmargin\dghspace180\dgvspace180\dghorigin450\dgvorigin0\dghshow1\dgvshow1 +\jexpand\viewkind5\viewscale100\pgbrdrhead\pgbrdrfoot\splytwnine\ftnlytwnine\htmautsp\nolnhtadjtbl\useltbaln\alntblind\lytcalctblwd\lyttblrtgr\lnbrkrule\nobrkwrptbl\snaptogridincell\allowfieldendsel\wrppunct +\asianbrkrule\rsidroot6097384\newtblstyruls\nogrowautofit\usenormstyforlist\noindnmbrts\felnbrelev\nocxsptable\indrlsweleven\noafcnsttbl\afelev\utinl\hwelev\spltpgpar\notcvasp\notbrkcnstfrctbl\notvatxbx\krnprsnet\cachedcolbal \nouicompat \fet0 +{\*\wgrffmtfilter 2450}\nofeaturethrottle1\ilfomacatclnup0{\*\ftnsep \ltrpar \pard\plain \ltrpar\ql \li0\ri0\widctlpar\wrapdefault\aspalpha\aspnum\faauto\adjustright\rin0\lin0\itap0 \rtlch\fcs1 \af46\afs24\alang1081 \ltrch\fcs0 +\fs24\lang1033\langfe2052\loch\af43\hich\af43\dbch\af45\cgrid\langnp1033\langfenp2052 {\rtlch\fcs1 \af46 \ltrch\fcs0 \insrsid16671749 \chftnsep +\par }}{\*\ftnsepc \ltrpar \pard\plain \ltrpar\ql \li0\ri0\widctlpar\wrapdefault\aspalpha\aspnum\faauto\adjustright\rin0\lin0\itap0 \rtlch\fcs1 \af46\afs24\alang1081 \ltrch\fcs0 +\fs24\lang1033\langfe2052\loch\af43\hich\af43\dbch\af45\cgrid\langnp1033\langfenp2052 {\rtlch\fcs1 \af46 \ltrch\fcs0 \insrsid16671749 \chftnsepc +\par }}{\*\aftnsep \ltrpar \pard\plain \ltrpar\ql \li0\ri0\widctlpar\wrapdefault\aspalpha\aspnum\faauto\adjustright\rin0\lin0\itap0 \rtlch\fcs1 \af46\afs24\alang1081 \ltrch\fcs0 +\fs24\lang1033\langfe2052\loch\af43\hich\af43\dbch\af45\cgrid\langnp1033\langfenp2052 {\rtlch\fcs1 \af46 \ltrch\fcs0 \insrsid16671749 \chftnsep +\par }}{\*\aftnsepc \ltrpar \pard\plain \ltrpar\ql \li0\ri0\widctlpar\wrapdefault\aspalpha\aspnum\faauto\adjustright\rin0\lin0\itap0 \rtlch\fcs1 \af46\afs24\alang1081 \ltrch\fcs0 +\fs24\lang1033\langfe2052\loch\af43\hich\af43\dbch\af45\cgrid\langnp1033\langfenp2052 {\rtlch\fcs1 \af46 \ltrch\fcs0 \insrsid16671749 \chftnsepc +\par }}\ltrpar \sectd \ltrsect\linex0\headery0\footery0\endnhere\sectunlocked1\sectdefaultcl\sftnbj {\*\pnseclvl1\pnucrm\pnstart1\pnindent720\pnhang {\pntxta .}}{\*\pnseclvl2\pnucltr\pnstart1\pnindent720\pnhang {\pntxta .}}{\*\pnseclvl3 +\pndec\pnstart1\pnindent720\pnhang {\pntxta .}}{\*\pnseclvl4\pnlcltr\pnstart1\pnindent720\pnhang {\pntxta )}}{\*\pnseclvl5\pndec\pnstart1\pnindent720\pnhang {\pntxtb (}{\pntxta )}}{\*\pnseclvl6\pnlcltr\pnstart1\pnindent720\pnhang {\pntxtb (}{\pntxta )}} +{\*\pnseclvl7\pnlcrm\pnstart1\pnindent720\pnhang {\pntxtb (}{\pntxta )}}{\*\pnseclvl8\pnlcltr\pnstart1\pnindent720\pnhang {\pntxtb (}{\pntxta )}}{\*\pnseclvl9\pnlcrm\pnstart1\pnindent720\pnhang {\pntxtb (}{\pntxta )}}\pard\plain \ltrpar +\qc \li0\ri0\widctlpar\wrapdefault\aspalpha\aspnum\faauto\adjustright\rin0\lin0\itap0 \rtlch\fcs1 \af46\afs24\alang1081 \ltrch\fcs0 \fs24\lang1033\langfe2052\loch\af43\hich\af43\dbch\af45\cgrid\langnp1033\langfenp2052 {\rtlch\fcs1 \af46\afs30 \ltrch\fcs0 +\fs30\insrsid16671749 \hich\af43\dbch\af45\loch\f43 A Text Extraction Test Document}{\rtlch\fcs1 \af46\afs30 \ltrch\fcs0 \fs30\insrsid6097384 +\par }{\rtlch\fcs1 \af46\afs20 \ltrch\fcs0 \fs20\insrsid16671749 \hich\af43\dbch\af45\loch\f43 for}{\rtlch\fcs1 \af46\afs20 \ltrch\fcs0 \fs20\insrsid6097384 +\par }{\rtlch\fcs1 \af46\afs30 \ltrch\fcs0 \fs30\insrsid16671749 \hich\af43\dbch\af45\loch\f43 DSpace}{\rtlch\fcs1 \af46\afs30 \ltrch\fcs0 \fs30\insrsid6097384 +\par +\par }\pard \ltrpar\ql \li0\ri0\widctlpar\wrapdefault\aspalpha\aspnum\faauto\adjustright\rin0\lin0\itap0 {\rtlch\fcs1 \af46 \ltrch\fcs0 \insrsid16671749 \hich\af43\dbch\af45\loch\f43 +This is a text. For the next sixty seconds this software will conduct a test of the DSpace text extraction facility. This is only a text.}{\rtlch\fcs1 \af46 \ltrch\fcs0 \insrsid6097384 +\par +\par }{\rtlch\fcs1 \af46 \ltrch\fcs0 \insrsid16671749 \hich\af43\dbch\af45\loch\f43 This is a paragraph that followed the first that lived in the \hich\af43\dbch\af45\loch\f43 document that Jack built.}{\rtlch\fcs1 \af46 \ltrch\fcs0 \insrsid6097384 +\par +\par }{\rtlch\fcs1 \af46 \ltrch\fcs0 \insrsid16671749 \hich\af43\dbch\af45\loch\f43 Lorem ipsum dolor sit amet. The quick brown fox jumped over the lazy dog. Yow! Are we having fun yet?}{\rtlch\fcs1 \af46 \ltrch\fcs0 \insrsid6097384 +\par +\par }{\rtlch\fcs1 \af46 \ltrch\fcs0 \insrsid16671749 \hich\af43\dbch\af45\loch\f43 This has been a test of the DSpace text extraction system. In the event of actual content you would care what is written he\hich\af43\dbch\af45\loch\f43 re.}{\rtlch\fcs1 +\af46 \ltrch\fcs0 \cs16\super\insrsid16671749 \chftn {\footnote \ltrpar \pard\plain \ltrpar\s24\ql \fi-339\li339\ri0\widctlpar\noline\wrapdefault\aspalpha\aspnum\faauto\adjustright\rin0\lin339\itap0 \rtlch\fcs1 \af46\afs20\alang1081 \ltrch\fcs0 +\fs20\lang1033\langfe2052\loch\af43\hich\af43\dbch\af45\cgrid\langnp1033\langfenp2052 {\rtlch\fcs1 \af46 \ltrch\fcs0 \insrsid16671749 \chftn \tab \hich\af43\dbch\af45\loch\f43 Tip o\hich\f43 \rquote \loch\f43 + the hat to the U.S. Emergency Broadcast System for the format that I have irreverently borrowed.}}}{\rtlch\fcs1 \af46 \ltrch\fcs0 \insrsid6097384 +\par }{\*\themedata 504b030414000600080000002100e9de0fbfff0000001c020000130000005b436f6e74656e745f54797065735d2e786d6cac91cb4ec3301045f748fc83e52d4a +9cb2400825e982c78ec7a27cc0c8992416c9d8b2a755fbf74cd25442a820166c2cd933f79e3be372bd1f07b5c3989ca74aaff2422b24eb1b475da5df374fd9ad +5689811a183c61a50f98f4babebc2837878049899a52a57be670674cb23d8e90721f90a4d2fa3802cb35762680fd800ecd7551dc18eb899138e3c943d7e503b6 +b01d583deee5f99824e290b4ba3f364eac4a430883b3c092d4eca8f946c916422ecab927f52ea42b89a1cd59c254f919b0e85e6535d135a8de20f20b8c12c3b0 +0c895fcf6720192de6bf3b9e89ecdbd6596cbcdd8eb28e7c365ecc4ec1ff1460f53fe813d3cc7f5b7f020000ffff0300504b030414000600080000002100a5d6 +a7e7c0000000360100000b0000005f72656c732f2e72656c73848fcf6ac3300c87ef85bd83d17d51d2c31825762fa590432fa37d00e1287f68221bdb1bebdb4f +c7060abb0884a4eff7a93dfeae8bf9e194e720169aaa06c3e2433fcb68e1763dbf7f82c985a4a725085b787086a37bdbb55fbc50d1a33ccd311ba548b6309512 +0f88d94fbc52ae4264d1c910d24a45db3462247fa791715fd71f989e19e0364cd3f51652d73760ae8fa8c9ffb3c330cc9e4fc17faf2ce545046e37944c69e462 +a1a82fe353bd90a865aad41ed0b5b8f9d6fd010000ffff0300504b0304140006000800000021006b799616830000008a0000001c0000007468656d652f746865 +6d652f7468656d654d616e616765722e786d6c0ccc4d0ac3201040e17da17790d93763bb284562b2cbaebbf600439c1a41c7a0d29fdbd7e5e38337cedf14d59b +4b0d592c9c070d8a65cd2e88b7f07c2ca71ba8da481cc52c6ce1c715e6e97818c9b48d13df49c873517d23d59085adb5dd20d6b52bd521ef2cdd5eb9246a3d8b +4757e8d3f729e245eb2b260a0238fd010000ffff0300504b030414000600080000002100b6f4679893070000c9200000160000007468656d652f7468656d652f +7468656d65312e786d6cec59cd8b1bc915bf07f23f347d97f5d5ad8fc1f2a24fcfda33b6b164873dd648a5eef2547789aad28cc56208de532e81c026e49085bd +ed21842cecc22eb9e48f31d8249b3f22afaa5bdd5552c99e191c3061463074977eefd5afde7bf5de53d5ddcf5e26d4bbc05c1096f6fcfa9d9aefe174ce16248d +7afeb3d9a4d2f13d2151ba4094a5b8e76fb0f03fbbf7eb5fdd454732c609f6403e1547a8e7c752ae8eaa5531876124eeb0154ee1bb25e30992f0caa3ea82a34b +d09bd06aa3566b55134452df4b51026a1f2f97648ebd9952e9dfdb2a1f53784da5500373caa74a35b6243476715e5708b11143cabd0b447b3eccb3609733fc52 +fa1e4542c2173dbfa6fffceabdbb5574940b517940d6909be8bf5c2e17589c37f49c3c3a2b260d823068f50bfd1a40e53e6edc1eb7c6ad429f06a0f91c569a71 +b175b61bc320c71aa0ecd1a17bd41e35eb16ded0dfdce3dc0fd5c7c26b50a63fd8c34f2643b0a285d7a00c1feee1c3417730b2f56b50866fede1dbb5fe28685b +fa3528a6243ddf43d7c25673b85d6d0159327aec8477c360d26ee4ca4b144443115d6a8a254be5a1584bd00bc6270050408a24493db959e1259a43140f112567 +9c7827248a21f056286502866b8ddaa4d684ffea13e827ed5174849121ad780113b137a4f87862cec94af6fc07a0d537206f7ffef9cdeb1fdfbcfee9cd575fbd +79fdf77c6eadca923b466964cafdf2dd1ffef3cd6fbd7ffff0ed2f5fff319b7a172f4cfcbbbffdeedd3ffef93ef5b0e2d2146ffff4fdbb1fbf7ffbe7dfffebaf +5f3bb4f7393a33e1339260e13dc297de5396c0021dfcf119bf9ec42c46c494e8a791402952b338f48f656ca11f6d10450edc00db767cce21d5b880f7d72f2cc2 +d398af2571687c182716f094313a60dc6985876a2ec3ccb3751ab927e76b13f714a10bd7dc43945a5e1eaf579063894be530c616cd2714a5124538c5d253dfb1 +738c1dabfb8210cbaea764ce99604be97d41bc01224e93ccc899154da5d03149c02f1b1741f0b7659bd3e7de8051d7aa47f8c246c2de40d4417e86a965c6fb68 +2d51e252394309350d7e8264ec2239ddf0b9891b0b099e8e3065de78818570c93ce6b05ec3e90f21cdb8dd7e4a37898de4929cbb749e20c64ce4889d0f6394ac +5cd829496313fbb938871045de13265df05366ef10f50e7e40e941773f27d872f787b3c133c8b026a53240d4376beef0e57dccacf89d6ee8126157aae9f3c44a +b17d4e9cd131584756689f604cd1255a60ec3dfbdcc160c05696cd4bd20f62c82ac7d815580f901dabea3dc5027a25d5dcece7c91322ac909de2881de073bad9 +493c1b9426881fd2fc08bc6eda7c0ca52e7105c0633a3f37818f08f480102f4ea33c16a0c308ee835a9fc4c82a60ea5db8e375c32dff5d658fc1be7c61d1b8c2 +be04197c6d1948eca6cc7b6d3343d49aa00c9819822ec3956e41c4727f29a28aab165b3be596f6a62ddd00dd91d5f42424fd6007b4d3fb84ffbbde073a8cb77f +f9c6b10f3e4ebfe3566c25ab6b763a8792c9f14e7f7308b7dbd50c195f904fbfa919a175fa04431dd9cf58b73dcd6d4fe3ffdff73487f6f36d2773a8dfb8ed64 +7ce8306e3b99fc70e5e3743265f3027d8d3af0c80e7af4b14f72f0d46749289dca0dc527421ffc08f83db398c0a092d3279eb838055cc5f0a8ca1c4c60e1228e +b48cc799fc0d91f134462b381daafb4a492472d591f0564cc0a1911e76ea5678ba4e4ed9223becacd7d5c16656590592e5782d2cc6e1a04a66e856bb3cc02bd4 +6bb6913e68dd1250b2d721614c6693683a48b4b783ca48fa58178ce620a157f65158741d2c3a4afdd6557b2c805ae115f8c1edc1cff49e1f06200242701e07cd +f942f92973f5d6bbda991fd3d3878c69450034d8db08283ddd555c0f2e4fad2e0bb52b78da2261849b4d425b46377822869fc17974aad1abd0b8aeafbba54b2d +7aca147a3e08ad9246bbf33e1637f535c8ede6069a9a9982a6de65cf6f35430899395af5fc251c1ac363b282d811ea3717a211dcbccc25cf36fc4d32cb8a0b39 +4222ce0cae934e960d122231f728497abe5a7ee1069aea1ca2b9d51b90103e59725d482b9f1a3970baed64bc5ce2b934dd6e8c284b67af90e1b35ce1fc568bdf +1cac24d91adc3d8d1797de195df3a708422c6cd795011744c0dd413db3e682c0655891c8caf8db294c79da356fa3740c65e388ae62945714339967709dca0b3a +faadb081f196af190c6a98242f8467912ab0a651ad6a5a548d8cc3c1aafb6121653923699635d3ca2aaa6abab39835c3b60cecd8f26645de60b53531e434b3c2 +67a97b37e576b7b96ea74f28aa0418bcb09fa3ea5ea12018d4cac92c6a8af17e1a56393b1fb56bc776811fa07695226164fdd656ed8edd8a1ae19c0e066f54f9 +416e376a6168b9ed2bb5a5f5adb979b1cdce5e40f2184197bba6526857c2c92e47d0104d754f92a50dd8222f65be35e0c95b73d2f3bfac85fd60d80887955a27 +1c57826650ab74c27eb3d20fc3667d1cd66ba341e31514161927f530bbb19fc00506dde4f7f67a7cefee3ed9ded1dc99b3a4caf4dd7c5513d777f7f5c6e1bb7b +8f40d2f9b2d598749bdd41abd26df627956034e854bac3d6a0326a0ddba3c9681876ba9357be77a1c141bf390c5ae34ea5551f0e2b41aba6e877ba9576d068f4 +8376bf330efaaff23606569ea58fdc16605ecdebde7f010000ffff0300504b0304140006000800000021000dd1909fb60000001b010000270000007468656d65 +2f7468656d652f5f72656c732f7468656d654d616e616765722e786d6c2e72656c73848f4d0ac2301484f78277086f6fd3ba109126dd88d0add40384e4350d36 +3f2451eced0dae2c082e8761be9969bb979dc9136332de3168aa1a083ae995719ac16db8ec8e4052164e89d93b64b060828e6f37ed1567914b284d262452282e +3198720e274a939cd08a54f980ae38a38f56e422a3a641c8bbd048f7757da0f19b017cc524bd62107bd5001996509affb3fd381a89672f1f165dfe514173d985 +0528a2c6cce0239baa4c04ca5bbabac4df000000ffff0300504b01022d0014000600080000002100e9de0fbfff0000001c020000130000000000000000000000 +0000000000005b436f6e74656e745f54797065735d2e786d6c504b01022d0014000600080000002100a5d6a7e7c0000000360100000b00000000000000000000 +000000300100005f72656c732f2e72656c73504b01022d00140006000800000021006b799616830000008a0000001c0000000000000000000000000019020000 +7468656d652f7468656d652f7468656d654d616e616765722e786d6c504b01022d0014000600080000002100b6f4679893070000c92000001600000000000000 +000000000000d60200007468656d652f7468656d652f7468656d65312e786d6c504b01022d00140006000800000021000dd1909fb60000001b01000027000000 +000000000000000000009d0a00007468656d652f7468656d652f5f72656c732f7468656d654d616e616765722e786d6c2e72656c73504b050600000000050005005d010000980b00000000} +{\*\colorschememapping 3c3f786d6c2076657273696f6e3d22312e302220656e636f64696e673d225554462d3822207374616e64616c6f6e653d22796573223f3e0d0a3c613a636c724d +617020786d6c6e733a613d22687474703a2f2f736368656d61732e6f70656e786d6c666f726d6174732e6f72672f64726177696e676d6c2f323030362f6d6169 +6e22206267313d226c743122207478313d22646b3122206267323d226c743222207478323d22646b322220616363656e74313d22616363656e74312220616363 +656e74323d22616363656e74322220616363656e74333d22616363656e74332220616363656e74343d22616363656e74342220616363656e74353d22616363656e74352220616363656e74363d22616363656e74362220686c696e6b3d22686c696e6b2220666f6c486c696e6b3d22666f6c486c696e6b222f3e} +{\*\latentstyles\lsdstimax376\lsdlockeddef0\lsdsemihiddendef0\lsdunhideuseddef0\lsdqformatdef0\lsdprioritydef99{\lsdlockedexcept \lsdqformat1 \lsdpriority0 \lsdlocked0 Normal;\lsdqformat1 \lsdpriority9 \lsdlocked0 heading 1; +\lsdsemihidden1 \lsdunhideused1 \lsdqformat1 \lsdpriority9 \lsdlocked0 heading 2;\lsdsemihidden1 \lsdunhideused1 \lsdqformat1 \lsdpriority9 \lsdlocked0 heading 3;\lsdsemihidden1 \lsdunhideused1 \lsdqformat1 \lsdpriority9 \lsdlocked0 heading 4; +\lsdsemihidden1 \lsdunhideused1 \lsdqformat1 \lsdpriority9 \lsdlocked0 heading 5;\lsdsemihidden1 \lsdunhideused1 \lsdqformat1 \lsdpriority9 \lsdlocked0 heading 6;\lsdsemihidden1 \lsdunhideused1 \lsdqformat1 \lsdpriority9 \lsdlocked0 heading 7; +\lsdsemihidden1 \lsdunhideused1 \lsdqformat1 \lsdpriority9 \lsdlocked0 heading 8;\lsdsemihidden1 \lsdunhideused1 \lsdqformat1 \lsdpriority9 \lsdlocked0 heading 9;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 index 1; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 index 2;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 index 3;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 index 4;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 index 5; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 index 6;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 index 7;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 index 8;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 index 9; +\lsdsemihidden1 \lsdunhideused1 \lsdpriority39 \lsdlocked0 toc 1;\lsdsemihidden1 \lsdunhideused1 \lsdpriority39 \lsdlocked0 toc 2;\lsdsemihidden1 \lsdunhideused1 \lsdpriority39 \lsdlocked0 toc 3; +\lsdsemihidden1 \lsdunhideused1 \lsdpriority39 \lsdlocked0 toc 4;\lsdsemihidden1 \lsdunhideused1 \lsdpriority39 \lsdlocked0 toc 5;\lsdsemihidden1 \lsdunhideused1 \lsdpriority39 \lsdlocked0 toc 6; +\lsdsemihidden1 \lsdunhideused1 \lsdpriority39 \lsdlocked0 toc 7;\lsdsemihidden1 \lsdunhideused1 \lsdpriority39 \lsdlocked0 toc 8;\lsdsemihidden1 \lsdunhideused1 \lsdpriority39 \lsdlocked0 toc 9;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Normal Indent; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 footnote text;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 annotation text;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 header;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 footer; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 index heading;\lsdsemihidden1 \lsdunhideused1 \lsdqformat1 \lsdpriority35 \lsdlocked0 caption;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 table of figures; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 envelope address;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 envelope return;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 footnote reference;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 annotation reference; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 line number;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 page number;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 endnote reference;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 endnote text; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 table of authorities;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 macro;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 toa heading;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 List; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 List Bullet;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 List Number;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 List 2;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 List 3; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 List 4;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 List 5;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 List Bullet 2;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 List Bullet 3; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 List Bullet 4;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 List Bullet 5;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 List Number 2;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 List Number 3; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 List Number 4;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 List Number 5;\lsdqformat1 \lsdpriority10 \lsdlocked0 Title;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Closing; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Signature;\lsdsemihidden1 \lsdunhideused1 \lsdpriority1 \lsdlocked0 Default Paragraph Font;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Body Text;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Body Text Indent; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 List Continue;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 List Continue 2;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 List Continue 3;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 List Continue 4; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 List Continue 5;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Message Header;\lsdqformat1 \lsdpriority11 \lsdlocked0 Subtitle;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Salutation; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Date;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Body Text First Indent;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Body Text First Indent 2;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Note Heading; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Body Text 2;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Body Text 3;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Body Text Indent 2;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Body Text Indent 3; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Block Text;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Hyperlink;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 FollowedHyperlink;\lsdqformat1 \lsdpriority22 \lsdlocked0 Strong; +\lsdqformat1 \lsdpriority20 \lsdlocked0 Emphasis;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Document Map;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Plain Text;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 E-mail Signature; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 HTML Top of Form;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 HTML Bottom of Form;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Normal (Web);\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 HTML Acronym; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 HTML Address;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 HTML Cite;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 HTML Code;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 HTML Definition; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 HTML Keyboard;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 HTML Preformatted;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 HTML Sample;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 HTML Typewriter; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 HTML Variable;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Normal Table;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 annotation subject;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 No List; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Outline List 1;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Outline List 2;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Outline List 3;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Simple 1; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Simple 2;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Simple 3;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Classic 1;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Classic 2; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Classic 3;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Classic 4;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Colorful 1;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Colorful 2; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Colorful 3;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Columns 1;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Columns 2;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Columns 3; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Columns 4;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Columns 5;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Grid 1;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Grid 2; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Grid 3;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Grid 4;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Grid 5;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Grid 6; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Grid 7;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Grid 8;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table List 1;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table List 2; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table List 3;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table List 4;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table List 5;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table List 6; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table List 7;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table List 8;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table 3D effects 1;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table 3D effects 2; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table 3D effects 3;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Contemporary;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Elegant;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Professional; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Subtle 1;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Subtle 2;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Web 1;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Web 2; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Web 3;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Balloon Text;\lsdpriority39 \lsdlocked0 Table Grid;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Table Theme;\lsdsemihidden1 \lsdlocked0 Placeholder Text; +\lsdqformat1 \lsdpriority1 \lsdlocked0 No Spacing;\lsdpriority60 \lsdlocked0 Light Shading;\lsdpriority61 \lsdlocked0 Light List;\lsdpriority62 \lsdlocked0 Light Grid;\lsdpriority63 \lsdlocked0 Medium Shading 1;\lsdpriority64 \lsdlocked0 Medium Shading 2; +\lsdpriority65 \lsdlocked0 Medium List 1;\lsdpriority66 \lsdlocked0 Medium List 2;\lsdpriority67 \lsdlocked0 Medium Grid 1;\lsdpriority68 \lsdlocked0 Medium Grid 2;\lsdpriority69 \lsdlocked0 Medium Grid 3;\lsdpriority70 \lsdlocked0 Dark List; +\lsdpriority71 \lsdlocked0 Colorful Shading;\lsdpriority72 \lsdlocked0 Colorful List;\lsdpriority73 \lsdlocked0 Colorful Grid;\lsdpriority60 \lsdlocked0 Light Shading Accent 1;\lsdpriority61 \lsdlocked0 Light List Accent 1; +\lsdpriority62 \lsdlocked0 Light Grid Accent 1;\lsdpriority63 \lsdlocked0 Medium Shading 1 Accent 1;\lsdpriority64 \lsdlocked0 Medium Shading 2 Accent 1;\lsdpriority65 \lsdlocked0 Medium List 1 Accent 1;\lsdsemihidden1 \lsdlocked0 Revision; +\lsdqformat1 \lsdpriority34 \lsdlocked0 List Paragraph;\lsdqformat1 \lsdpriority29 \lsdlocked0 Quote;\lsdqformat1 \lsdpriority30 \lsdlocked0 Intense Quote;\lsdpriority66 \lsdlocked0 Medium List 2 Accent 1;\lsdpriority67 \lsdlocked0 Medium Grid 1 Accent 1; +\lsdpriority68 \lsdlocked0 Medium Grid 2 Accent 1;\lsdpriority69 \lsdlocked0 Medium Grid 3 Accent 1;\lsdpriority70 \lsdlocked0 Dark List Accent 1;\lsdpriority71 \lsdlocked0 Colorful Shading Accent 1;\lsdpriority72 \lsdlocked0 Colorful List Accent 1; +\lsdpriority73 \lsdlocked0 Colorful Grid Accent 1;\lsdpriority60 \lsdlocked0 Light Shading Accent 2;\lsdpriority61 \lsdlocked0 Light List Accent 2;\lsdpriority62 \lsdlocked0 Light Grid Accent 2;\lsdpriority63 \lsdlocked0 Medium Shading 1 Accent 2; +\lsdpriority64 \lsdlocked0 Medium Shading 2 Accent 2;\lsdpriority65 \lsdlocked0 Medium List 1 Accent 2;\lsdpriority66 \lsdlocked0 Medium List 2 Accent 2;\lsdpriority67 \lsdlocked0 Medium Grid 1 Accent 2;\lsdpriority68 \lsdlocked0 Medium Grid 2 Accent 2; +\lsdpriority69 \lsdlocked0 Medium Grid 3 Accent 2;\lsdpriority70 \lsdlocked0 Dark List Accent 2;\lsdpriority71 \lsdlocked0 Colorful Shading Accent 2;\lsdpriority72 \lsdlocked0 Colorful List Accent 2;\lsdpriority73 \lsdlocked0 Colorful Grid Accent 2; +\lsdpriority60 \lsdlocked0 Light Shading Accent 3;\lsdpriority61 \lsdlocked0 Light List Accent 3;\lsdpriority62 \lsdlocked0 Light Grid Accent 3;\lsdpriority63 \lsdlocked0 Medium Shading 1 Accent 3;\lsdpriority64 \lsdlocked0 Medium Shading 2 Accent 3; +\lsdpriority65 \lsdlocked0 Medium List 1 Accent 3;\lsdpriority66 \lsdlocked0 Medium List 2 Accent 3;\lsdpriority67 \lsdlocked0 Medium Grid 1 Accent 3;\lsdpriority68 \lsdlocked0 Medium Grid 2 Accent 3;\lsdpriority69 \lsdlocked0 Medium Grid 3 Accent 3; +\lsdpriority70 \lsdlocked0 Dark List Accent 3;\lsdpriority71 \lsdlocked0 Colorful Shading Accent 3;\lsdpriority72 \lsdlocked0 Colorful List Accent 3;\lsdpriority73 \lsdlocked0 Colorful Grid Accent 3;\lsdpriority60 \lsdlocked0 Light Shading Accent 4; +\lsdpriority61 \lsdlocked0 Light List Accent 4;\lsdpriority62 \lsdlocked0 Light Grid Accent 4;\lsdpriority63 \lsdlocked0 Medium Shading 1 Accent 4;\lsdpriority64 \lsdlocked0 Medium Shading 2 Accent 4;\lsdpriority65 \lsdlocked0 Medium List 1 Accent 4; +\lsdpriority66 \lsdlocked0 Medium List 2 Accent 4;\lsdpriority67 \lsdlocked0 Medium Grid 1 Accent 4;\lsdpriority68 \lsdlocked0 Medium Grid 2 Accent 4;\lsdpriority69 \lsdlocked0 Medium Grid 3 Accent 4;\lsdpriority70 \lsdlocked0 Dark List Accent 4; +\lsdpriority71 \lsdlocked0 Colorful Shading Accent 4;\lsdpriority72 \lsdlocked0 Colorful List Accent 4;\lsdpriority73 \lsdlocked0 Colorful Grid Accent 4;\lsdpriority60 \lsdlocked0 Light Shading Accent 5;\lsdpriority61 \lsdlocked0 Light List Accent 5; +\lsdpriority62 \lsdlocked0 Light Grid Accent 5;\lsdpriority63 \lsdlocked0 Medium Shading 1 Accent 5;\lsdpriority64 \lsdlocked0 Medium Shading 2 Accent 5;\lsdpriority65 \lsdlocked0 Medium List 1 Accent 5;\lsdpriority66 \lsdlocked0 Medium List 2 Accent 5; +\lsdpriority67 \lsdlocked0 Medium Grid 1 Accent 5;\lsdpriority68 \lsdlocked0 Medium Grid 2 Accent 5;\lsdpriority69 \lsdlocked0 Medium Grid 3 Accent 5;\lsdpriority70 \lsdlocked0 Dark List Accent 5;\lsdpriority71 \lsdlocked0 Colorful Shading Accent 5; +\lsdpriority72 \lsdlocked0 Colorful List Accent 5;\lsdpriority73 \lsdlocked0 Colorful Grid Accent 5;\lsdpriority60 \lsdlocked0 Light Shading Accent 6;\lsdpriority61 \lsdlocked0 Light List Accent 6;\lsdpriority62 \lsdlocked0 Light Grid Accent 6; +\lsdpriority63 \lsdlocked0 Medium Shading 1 Accent 6;\lsdpriority64 \lsdlocked0 Medium Shading 2 Accent 6;\lsdpriority65 \lsdlocked0 Medium List 1 Accent 6;\lsdpriority66 \lsdlocked0 Medium List 2 Accent 6; +\lsdpriority67 \lsdlocked0 Medium Grid 1 Accent 6;\lsdpriority68 \lsdlocked0 Medium Grid 2 Accent 6;\lsdpriority69 \lsdlocked0 Medium Grid 3 Accent 6;\lsdpriority70 \lsdlocked0 Dark List Accent 6;\lsdpriority71 \lsdlocked0 Colorful Shading Accent 6; +\lsdpriority72 \lsdlocked0 Colorful List Accent 6;\lsdpriority73 \lsdlocked0 Colorful Grid Accent 6;\lsdqformat1 \lsdpriority19 \lsdlocked0 Subtle Emphasis;\lsdqformat1 \lsdpriority21 \lsdlocked0 Intense Emphasis; +\lsdqformat1 \lsdpriority31 \lsdlocked0 Subtle Reference;\lsdqformat1 \lsdpriority32 \lsdlocked0 Intense Reference;\lsdqformat1 \lsdpriority33 \lsdlocked0 Book Title;\lsdsemihidden1 \lsdunhideused1 \lsdpriority37 \lsdlocked0 Bibliography; +\lsdsemihidden1 \lsdunhideused1 \lsdqformat1 \lsdpriority39 \lsdlocked0 TOC Heading;\lsdpriority41 \lsdlocked0 Plain Table 1;\lsdpriority42 \lsdlocked0 Plain Table 2;\lsdpriority43 \lsdlocked0 Plain Table 3;\lsdpriority44 \lsdlocked0 Plain Table 4; +\lsdpriority45 \lsdlocked0 Plain Table 5;\lsdpriority40 \lsdlocked0 Grid Table Light;\lsdpriority46 \lsdlocked0 Grid Table 1 Light;\lsdpriority47 \lsdlocked0 Grid Table 2;\lsdpriority48 \lsdlocked0 Grid Table 3;\lsdpriority49 \lsdlocked0 Grid Table 4; +\lsdpriority50 \lsdlocked0 Grid Table 5 Dark;\lsdpriority51 \lsdlocked0 Grid Table 6 Colorful;\lsdpriority52 \lsdlocked0 Grid Table 7 Colorful;\lsdpriority46 \lsdlocked0 Grid Table 1 Light Accent 1;\lsdpriority47 \lsdlocked0 Grid Table 2 Accent 1; +\lsdpriority48 \lsdlocked0 Grid Table 3 Accent 1;\lsdpriority49 \lsdlocked0 Grid Table 4 Accent 1;\lsdpriority50 \lsdlocked0 Grid Table 5 Dark Accent 1;\lsdpriority51 \lsdlocked0 Grid Table 6 Colorful Accent 1; +\lsdpriority52 \lsdlocked0 Grid Table 7 Colorful Accent 1;\lsdpriority46 \lsdlocked0 Grid Table 1 Light Accent 2;\lsdpriority47 \lsdlocked0 Grid Table 2 Accent 2;\lsdpriority48 \lsdlocked0 Grid Table 3 Accent 2; +\lsdpriority49 \lsdlocked0 Grid Table 4 Accent 2;\lsdpriority50 \lsdlocked0 Grid Table 5 Dark Accent 2;\lsdpriority51 \lsdlocked0 Grid Table 6 Colorful Accent 2;\lsdpriority52 \lsdlocked0 Grid Table 7 Colorful Accent 2; +\lsdpriority46 \lsdlocked0 Grid Table 1 Light Accent 3;\lsdpriority47 \lsdlocked0 Grid Table 2 Accent 3;\lsdpriority48 \lsdlocked0 Grid Table 3 Accent 3;\lsdpriority49 \lsdlocked0 Grid Table 4 Accent 3; +\lsdpriority50 \lsdlocked0 Grid Table 5 Dark Accent 3;\lsdpriority51 \lsdlocked0 Grid Table 6 Colorful Accent 3;\lsdpriority52 \lsdlocked0 Grid Table 7 Colorful Accent 3;\lsdpriority46 \lsdlocked0 Grid Table 1 Light Accent 4; +\lsdpriority47 \lsdlocked0 Grid Table 2 Accent 4;\lsdpriority48 \lsdlocked0 Grid Table 3 Accent 4;\lsdpriority49 \lsdlocked0 Grid Table 4 Accent 4;\lsdpriority50 \lsdlocked0 Grid Table 5 Dark Accent 4; +\lsdpriority51 \lsdlocked0 Grid Table 6 Colorful Accent 4;\lsdpriority52 \lsdlocked0 Grid Table 7 Colorful Accent 4;\lsdpriority46 \lsdlocked0 Grid Table 1 Light Accent 5;\lsdpriority47 \lsdlocked0 Grid Table 2 Accent 5; +\lsdpriority48 \lsdlocked0 Grid Table 3 Accent 5;\lsdpriority49 \lsdlocked0 Grid Table 4 Accent 5;\lsdpriority50 \lsdlocked0 Grid Table 5 Dark Accent 5;\lsdpriority51 \lsdlocked0 Grid Table 6 Colorful Accent 5; +\lsdpriority52 \lsdlocked0 Grid Table 7 Colorful Accent 5;\lsdpriority46 \lsdlocked0 Grid Table 1 Light Accent 6;\lsdpriority47 \lsdlocked0 Grid Table 2 Accent 6;\lsdpriority48 \lsdlocked0 Grid Table 3 Accent 6; +\lsdpriority49 \lsdlocked0 Grid Table 4 Accent 6;\lsdpriority50 \lsdlocked0 Grid Table 5 Dark Accent 6;\lsdpriority51 \lsdlocked0 Grid Table 6 Colorful Accent 6;\lsdpriority52 \lsdlocked0 Grid Table 7 Colorful Accent 6; +\lsdpriority46 \lsdlocked0 List Table 1 Light;\lsdpriority47 \lsdlocked0 List Table 2;\lsdpriority48 \lsdlocked0 List Table 3;\lsdpriority49 \lsdlocked0 List Table 4;\lsdpriority50 \lsdlocked0 List Table 5 Dark; +\lsdpriority51 \lsdlocked0 List Table 6 Colorful;\lsdpriority52 \lsdlocked0 List Table 7 Colorful;\lsdpriority46 \lsdlocked0 List Table 1 Light Accent 1;\lsdpriority47 \lsdlocked0 List Table 2 Accent 1;\lsdpriority48 \lsdlocked0 List Table 3 Accent 1; +\lsdpriority49 \lsdlocked0 List Table 4 Accent 1;\lsdpriority50 \lsdlocked0 List Table 5 Dark Accent 1;\lsdpriority51 \lsdlocked0 List Table 6 Colorful Accent 1;\lsdpriority52 \lsdlocked0 List Table 7 Colorful Accent 1; +\lsdpriority46 \lsdlocked0 List Table 1 Light Accent 2;\lsdpriority47 \lsdlocked0 List Table 2 Accent 2;\lsdpriority48 \lsdlocked0 List Table 3 Accent 2;\lsdpriority49 \lsdlocked0 List Table 4 Accent 2; +\lsdpriority50 \lsdlocked0 List Table 5 Dark Accent 2;\lsdpriority51 \lsdlocked0 List Table 6 Colorful Accent 2;\lsdpriority52 \lsdlocked0 List Table 7 Colorful Accent 2;\lsdpriority46 \lsdlocked0 List Table 1 Light Accent 3; +\lsdpriority47 \lsdlocked0 List Table 2 Accent 3;\lsdpriority48 \lsdlocked0 List Table 3 Accent 3;\lsdpriority49 \lsdlocked0 List Table 4 Accent 3;\lsdpriority50 \lsdlocked0 List Table 5 Dark Accent 3; +\lsdpriority51 \lsdlocked0 List Table 6 Colorful Accent 3;\lsdpriority52 \lsdlocked0 List Table 7 Colorful Accent 3;\lsdpriority46 \lsdlocked0 List Table 1 Light Accent 4;\lsdpriority47 \lsdlocked0 List Table 2 Accent 4; +\lsdpriority48 \lsdlocked0 List Table 3 Accent 4;\lsdpriority49 \lsdlocked0 List Table 4 Accent 4;\lsdpriority50 \lsdlocked0 List Table 5 Dark Accent 4;\lsdpriority51 \lsdlocked0 List Table 6 Colorful Accent 4; +\lsdpriority52 \lsdlocked0 List Table 7 Colorful Accent 4;\lsdpriority46 \lsdlocked0 List Table 1 Light Accent 5;\lsdpriority47 \lsdlocked0 List Table 2 Accent 5;\lsdpriority48 \lsdlocked0 List Table 3 Accent 5; +\lsdpriority49 \lsdlocked0 List Table 4 Accent 5;\lsdpriority50 \lsdlocked0 List Table 5 Dark Accent 5;\lsdpriority51 \lsdlocked0 List Table 6 Colorful Accent 5;\lsdpriority52 \lsdlocked0 List Table 7 Colorful Accent 5; +\lsdpriority46 \lsdlocked0 List Table 1 Light Accent 6;\lsdpriority47 \lsdlocked0 List Table 2 Accent 6;\lsdpriority48 \lsdlocked0 List Table 3 Accent 6;\lsdpriority49 \lsdlocked0 List Table 4 Accent 6; +\lsdpriority50 \lsdlocked0 List Table 5 Dark Accent 6;\lsdpriority51 \lsdlocked0 List Table 6 Colorful Accent 6;\lsdpriority52 \lsdlocked0 List Table 7 Colorful Accent 6;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Mention; +\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Smart Hyperlink;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Hashtag;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Unresolved Mention;\lsdsemihidden1 \lsdunhideused1 \lsdlocked0 Smart Link;}}{\*\datastore 01050000 +02000000180000004d73786d6c322e534158584d4c5265616465722e362e3000000000000000000000060000 +d0cf11e0a1b11ae1000000000000000000000000000000003e000300feff090006000000000000000000000001000000010000000000000000100000feffffff00000000feffffff0000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +fffffffffffffffffdfffffffeffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffff52006f006f007400200045006e00740072007900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000016000500ffffffffffffffffffffffff0c6ad98892f1d411a65f0040963251e5000000000000000000000000d0af +77916744d801feffffff00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000ffffffffffffffffffffffff00000000000000000000000000000000000000000000000000000000 +00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000ffffffffffffffffffffffff0000000000000000000000000000000000000000000000000000 +000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000ffffffffffffffffffffffff000000000000000000000000000000000000000000000000 +0000000000000000000000000000000000000000000000000105000000000000}} \ No newline at end of file diff --git a/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.txt b/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.txt new file mode 100644 index 0000000000..edd9160b1d --- /dev/null +++ b/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.txt @@ -0,0 +1,13 @@ +A Text Extraction Test Document +for +DSpace + +This is a text. For the next sixty seconds this software will conduct a test of the DSpace text extraction facility. This is only a text. + +This is a paragraph that followed the first that lived in the document that Jack built. + +Lorem ipsum dolor sit amet. The quick brown fox jumped over the lazy dog. Yow! Are we having fun yet? + +This has been a test of the DSpace text extraction system. In the event of actual content you would care what is written here. + +Tip o’ the hat to the U.S. Emergency Broadcast System for the format that I have irreverently borrowed. \ No newline at end of file diff --git a/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.xls b/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.xls new file mode 100644 index 0000000000..1ebc20bc38 Binary files /dev/null and b/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.xls differ diff --git a/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.xlsx b/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.xlsx new file mode 100644 index 0000000000..47e0f7387f Binary files /dev/null and b/dspace-api/src/test/resources/org/dspace/app/mediafilter/test.xlsx differ diff --git a/dspace-iiif/pom.xml b/dspace-iiif/pom.xml index c8f64c6f04..e606523182 100644 --- a/dspace-iiif/pom.xml +++ b/dspace-iiif/pom.xml @@ -45,6 +45,13 @@ org.springframework.boot spring-boot-starter-web ${spring-boot.version} + + + + org.hibernate.validator + hibernate-validator + +     org.springframework.boot diff --git a/dspace-rest/pom.xml b/dspace-rest/pom.xml index 4433cffd33..5377dab901 100644 --- a/dspace-rest/pom.xml +++ b/dspace-rest/pom.xml @@ -102,6 +102,17 @@ com.fasterxml.jackson.module jackson-module-jaxb-annotations ${jackson.version} + + + + jakarta.activation + jakarta.activation-api + + + jakarta.xml.bind + jakarta.xml.bind-api + + diff --git a/dspace-server-webapp/pom.xml b/dspace-server-webapp/pom.xml index 20e6602347..4677366a2c 100644 --- a/dspace-server-webapp/pom.xml +++ b/dspace-server-webapp/pom.xml @@ -260,6 +260,13 @@ org.springframework.boot spring-boot-starter-web ${spring-boot.version} + + + + org.hibernate.validator + hibernate-validator + +     org.springframework.boot @@ -364,13 +371,12 @@ dspace-services + + org.dspace dspace-iiif - - - org.dspace dspace-oai @@ -416,6 +422,32 @@ ${nimbus-jose-jwt.version} + + org.apache.solr + solr-solrj + ${solr.client.version} + + + + + org.springframework.boot + spring-boot-starter-cache + ${spring-boot.version} + + + + javax.cache + cache-api + 1.1.0 + + + + org.ehcache + ehcache + ${ehcache.version} + + org.springframework.boot @@ -488,50 +520,6 @@ - - - org.springframework.boot - spring-boot-starter-cache - ${spring-boot.version} - - - - javax.cache - cache-api - 1.1.0 - - - - org.ehcache - ehcache - ${ehcache.version} - - - org.apache.solr - solr-cell - test - - - - org.apache.commons - commons-text - - - - org.eclipse.jetty - jetty-http - - - org.eclipse.jetty - jetty-io - - - org.eclipse.jetty - jetty-util - - - org.apache.lucene lucene-analyzers-icu diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/utils/HttpHeadersInitializer.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/utils/HttpHeadersInitializer.java index 22ae8ad3d9..ffe2acc2e0 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/utils/HttpHeadersInitializer.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/utils/HttpHeadersInitializer.java @@ -9,6 +9,7 @@ package org.dspace.app.rest.utils; import static java.util.Objects.isNull; import static java.util.Objects.nonNull; +import static javax.mail.internet.MimeUtility.encodeText; import java.io.IOException; import java.util.Arrays; @@ -163,7 +164,8 @@ public class HttpHeadersInitializer { } httpHeaders.put(CONTENT_DISPOSITION, Collections.singletonList(String.format(CONTENT_DISPOSITION_FORMAT, - disposition, fileName))); + disposition, + encodeText(fileName)))); log.debug("Content-Disposition : {}", disposition); // Content phase diff --git a/dspace-server-webapp/src/main/resources/application.properties b/dspace-server-webapp/src/main/resources/application.properties index c695fe2fba..91e2aa73ae 100644 --- a/dspace-server-webapp/src/main/resources/application.properties +++ b/dspace-server-webapp/src/main/resources/application.properties @@ -117,6 +117,8 @@ spring.main.allow-bean-definition-overriding = true ######################### # Spring Boot Logging levels # +# NOTE: The below settings can be uncommented to debug issues in Spring Boot/WebMVC. +# These "logging.level" settings will also override defaults in "logging.config" below. #logging.level.org.springframework.boot=DEBUG #logging.level.org.springframework.web=DEBUG #logging.level.org.hibernate=ERROR diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/BitstreamControllerIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/BitstreamControllerIT.java index ca3c05ec30..4b1124071a 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/BitstreamControllerIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/BitstreamControllerIT.java @@ -16,6 +16,8 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import java.io.InputStream; +import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import org.apache.commons.codec.CharEncoding; @@ -144,47 +146,48 @@ public class BitstreamControllerIT extends AbstractControllerIntegrationTest { String bitstreamContent = "ThisIsSomeDummyText"; - Bundle bundle1 = BundleBuilder.createBundle(context, publicItem1) + List bundles = new ArrayList(); + bundles.add(BundleBuilder.createBundle(context, publicItem1) .withName("TEST FIRST BUNDLE") - .build(); + .build()); Bitstream bitstream = null; try (InputStream is = IOUtils.toInputStream(bitstreamContent, CharEncoding.UTF_8)) { bitstream = BitstreamBuilder. - createBitstream(context, bundle1, is) + createBitstream(context, bundles.get(0), is) .withName("Bitstream") .withDescription("description") .withMimeType("text/plain") .build(); } - Bundle bundle2 = BundleBuilder.createBundle(context, publicItem1) + bundles.add(BundleBuilder.createBundle(context, publicItem1) .withName("TEST SECOND BUNDLE") .withBitstream(bitstream) - .build(); + .build()); context.restoreAuthSystemState(); + // While in DSpace code, Bundles are *unordered*, in Hibernate v5 + H2 v2.x, they are returned sorted by UUID. + // So, we reorder this list of created Bundles by UUID to get their expected return order. + // NOTE: Once on Hibernate v6, this might need "toString()" removed as it may sort UUIDs based on RFC 4412. + Comparator compareByUUID = Comparator.comparing(b -> b.getID().toString()); + bundles.sort(compareByUUID); + String token = getAuthToken(admin.getEmail(), password); + // Expect only the first Bundle to be returned getClient().perform(get("/api/core/bitstreams/" + bitstream.getID() + "/bundle") .param("projection", "full")) .andExpect(status().isOk()) .andExpect(content().contentType(contentType)) .andExpect(jsonPath("$", Matchers.is( - BundleMatcher.matchBundle(bundle1.getName(), - bundle1.getID(), - bundle1.getHandle(), - bundle1.getType(), - bundle1.getBitstreams()) - - ))).andExpect(jsonPath("$", Matchers.not( - BundleMatcher.matchBundle(bundle2.getName(), - bundle2.getID(), - bundle2.getHandle(), - bundle2.getType(), - bundle2.getBitstreams()) + BundleMatcher.matchBundle(bundles.get(0).getName(), + bundles.get(0).getID(), + bundles.get(0).getHandle(), + bundles.get(0).getType(), + bundles.get(0).getBitstreams()) ))); diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/BitstreamFormatRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/BitstreamFormatRestRepositoryIT.java index 48ad410d00..1a6cc29ca7 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/BitstreamFormatRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/BitstreamFormatRestRepositoryIT.java @@ -56,7 +56,7 @@ public class BitstreamFormatRestRepositoryIT extends AbstractControllerIntegrati @Autowired private BitstreamFormatConverter bitstreamFormatConverter; - private final int DEFAULT_AMOUNT_FORMATS = 80; + private final int DEFAULT_AMOUNT_FORMATS = 81; @Test public void findAllPaginationTest() throws Exception { diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/BitstreamRestControllerIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/BitstreamRestControllerIT.java index f07aae876f..efb7d1776d 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/BitstreamRestControllerIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/BitstreamRestControllerIT.java @@ -8,6 +8,7 @@ package org.dspace.app.rest; import static java.util.UUID.randomUUID; +import static javax.mail.internet.MimeUtility.encodeText; import static org.apache.commons.codec.CharEncoding.UTF_8; import static org.apache.commons.collections.CollectionUtils.isEmpty; import static org.apache.commons.io.IOUtils.toInputStream; @@ -293,6 +294,53 @@ public class BitstreamRestControllerIT extends AbstractControllerIntegrationTest checkNumberOfStatsRecords(bitstream, 0); } + @Test + public void testBitstreamName() throws Exception { + + context.turnOffAuthorisationSystem(); + + //** GIVEN ** + //1. A community-collection structure with one parent community and one collection + + parentCommunity = CommunityBuilder + .createCommunity(context) + .build(); + + Collection collection = CollectionBuilder + .createCollection(context, parentCommunity) + .build(); + + //2. A public item with a bitstream + + String bitstreamContent = "0123456789"; + String bitstreamName = "ภาษาไทย"; + + try (InputStream is = IOUtils.toInputStream(bitstreamContent, CharEncoding.UTF_8)) { + + Item item = ItemBuilder + .createItem(context, collection) + .build(); + + bitstream = BitstreamBuilder + .createBitstream(context, item, is) + .withName(bitstreamName) + .build(); + } + + context.restoreAuthSystemState(); + + //** WHEN ** + //We download the bitstream + getClient().perform(get("/api/core/bitstreams/" + bitstream.getID() + "/content")) + //** THEN ** + .andExpect(status().isOk()) + //We expect the content disposition to have the encoded bitstream name + .andExpect(header().string( + "Content-Disposition", + "attachment;filename=\"" + encodeText(bitstreamName) + "\"" + )); + } + @Test public void testBitstreamNotFound() throws Exception { getClient().perform(get("/api/core/bitstreams/" + UUID.randomUUID() + "/content")) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/BitstreamRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/BitstreamRestRepositoryIT.java index fd95e7f584..f9c1e469fc 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/BitstreamRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/BitstreamRestRepositoryIT.java @@ -20,6 +20,8 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import java.io.InputStream; +import java.util.Comparator; +import java.util.List; import java.util.UUID; import org.apache.commons.codec.CharEncoding; @@ -45,6 +47,7 @@ import org.dspace.content.Community; import org.dspace.content.Item; import org.dspace.content.service.BitstreamFormatService; import org.dspace.content.service.BitstreamService; +import org.dspace.content.service.ItemService; import org.dspace.core.Constants; import org.dspace.eperson.EPerson; import org.dspace.eperson.Group; @@ -67,6 +70,10 @@ public class BitstreamRestRepositoryIT extends AbstractControllerIntegrationTest @Autowired private GroupService groupService; + + @Autowired + private ItemService itemService; + @Test public void findAllTest() throws Exception { //We turn off the authorization system in order to create the structure as defined below @@ -1480,11 +1487,19 @@ public class BitstreamRestRepositoryIT extends AbstractControllerIntegrationTest .build(); } - Bundle secondBundle = BundleBuilder.createBundle(context, publicItem1) - .withName("second bundle") - .withBitstream(bitstream).build(); + // Add default content bundle to list of bundles + List bundles = itemService.getBundles(publicItem1, Constants.CONTENT_BUNDLE_NAME); - Bundle bundle = bitstream.getBundles().get(0); + // Add this bitstream to a second bundle & append to list of bundles + bundles.add(BundleBuilder.createBundle(context, publicItem1) + .withName("second bundle") + .withBitstream(bitstream).build()); + + // While in DSpace code, Bundles are *unordered*, in Hibernate v5 + H2 v2.x, they are returned sorted by UUID. + // So, we reorder this list of created Bundles by UUID to get their expected return order. + // NOTE: Once on Hibernate v6, this might need "toString()" removed as it may sort UUIDs based on RFC 4412. + Comparator compareByUUID = Comparator.comparing(b -> b.getID().toString()); + bundles.sort(compareByUUID); //Get bundle should contain the first bundle in the list getClient().perform(get("/api/core/bitstreams/" + bitstream.getID() + "/bundle")) @@ -1492,10 +1507,10 @@ public class BitstreamRestRepositoryIT extends AbstractControllerIntegrationTest .andExpect(content().contentType(contentType)) .andExpect(jsonPath("$", BundleMatcher.matchProperties( - bundle.getName(), - bundle.getID(), - bundle.getHandle(), - bundle.getType() + bundles.get(0).getName(), + bundles.get(0).getID(), + bundles.get(0).getHandle(), + bundles.get(0).getType() ) )); } diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/BundleRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/BundleRestRepositoryIT.java index 884d551582..96385095a2 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/BundleRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/BundleRestRepositoryIT.java @@ -397,26 +397,24 @@ public class BundleRestRepositoryIT extends AbstractControllerIntegrationTest { public void getBitstreamsForBundle() throws Exception { context.turnOffAuthorisationSystem(); + bundle1 = BundleBuilder.createBundle(context, item) + .withName("testname") + .build(); + String bitstreamContent = "Dummy content"; try (InputStream is = IOUtils.toInputStream(bitstreamContent, CharEncoding.UTF_8)) { - bitstream1 = BitstreamBuilder.createBitstream(context, item, is) + bitstream1 = BitstreamBuilder.createBitstream(context, item, is, bundle1.getName()) .withName("Bitstream") .withDescription("Description") .withMimeType("text/plain") .build(); - bitstream2 = BitstreamBuilder.createBitstream(context, item, is) + bitstream2 = BitstreamBuilder.createBitstream(context, item, is, bundle1.getName()) .withName("Bitstream2") .withDescription("Description2") .withMimeType("text/plain") .build(); } - bundle1 = BundleBuilder.createBundle(context, item) - .withName("testname") - .withBitstream(bitstream1) - .withBitstream(bitstream2) - .build(); - context.restoreAuthSystemState(); getClient().perform(get("/api/core/bundles/" + bundle1.getID() + "/bitstreams") @@ -465,26 +463,24 @@ public class BundleRestRepositoryIT extends AbstractControllerIntegrationTest { public void patchMoveBitstreams() throws Exception { context.turnOffAuthorisationSystem(); + bundle1 = BundleBuilder.createBundle(context, item) + .withName("testname") + .build(); + String bitstreamContent = "Dummy content"; try (InputStream is = IOUtils.toInputStream(bitstreamContent, CharEncoding.UTF_8)) { - bitstream1 = BitstreamBuilder.createBitstream(context, item, is) + bitstream1 = BitstreamBuilder.createBitstream(context, item, is, bundle1.getName()) .withName("Bitstream") .withDescription("Description") .withMimeType("text/plain") .build(); - bitstream2 = BitstreamBuilder.createBitstream(context, item, is) + bitstream2 = BitstreamBuilder.createBitstream(context, item, is, bundle1.getName()) .withName("Bitstream2") .withDescription("Description2") .withMimeType("text/plain") .build(); } - bundle1 = BundleBuilder.createBundle(context, item) - .withName("testname") - .withBitstream(bitstream1) - .withBitstream(bitstream2) - .build(); - context.restoreAuthSystemState(); getClient().perform(get("/api/core/bundles/" + bundle1.getID() + "/bitstreams") diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/CollectionRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/CollectionRestRepositoryIT.java index 92fdecf824..6ae33cb280 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/CollectionRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/CollectionRestRepositoryIT.java @@ -2421,9 +2421,11 @@ public class CollectionRestRepositoryIT extends AbstractControllerIntegrationTes .build(); List items = new ArrayList(); - // This comparator is used to sort our test Items by java.util.UUID (which sorts them based on the RFC - // and not based on String comparison, see also https://stackoverflow.com/a/51031298/3750035 ) - Comparator compareByUUID = Comparator.comparing(i -> i.getID()); + // Hibernate 5.x's org.hibernate.dialect.H2Dialect sorts UUIDs as if they are Strings. + // So, we must compare UUIDs as if they are strings. + // In Hibernate 6, the H2Dialect has been updated with native UUID type support, at which point + // we'd need to update the below comparator to compare them as java.util.UUID (which sorts based on RFC 4412). + Comparator compareByUUID = Comparator.comparing(i -> i.getID().toString()); Item item0 = ItemBuilder.createItem(context, collection).withTitle("Item 0").build(); items.add(item0); diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/EntityTypeRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/EntityTypeRestRepositoryIT.java index 93aacb60e4..740a2c0dc3 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/EntityTypeRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/EntityTypeRestRepositoryIT.java @@ -86,13 +86,13 @@ public class EntityTypeRestRepositoryIT extends AbstractEntityIntegrationTest { .andExpect(jsonPath("$._links.self.href", containsString("api/core/entitytypes"))) //We have 4 facets in the default configuration, they need to all be present in the embedded section .andExpect(jsonPath("$._embedded.entitytypes", containsInAnyOrder( - EntityTypeMatcher.matchEntityTypeEntry(entityTypeService.findByEntityType(context, - Constants.ENTITY_TYPE_NONE)), + EntityTypeMatcher.matchEntityTypeEntry(entityTypeService.findByEntityType(context, "Journal")), EntityTypeMatcher - .matchEntityTypeEntry(entityTypeService.findByEntityType(context, "Publication")), - EntityTypeMatcher.matchEntityTypeEntry(entityTypeService.findByEntityType(context, "Person")), - EntityTypeMatcher.matchEntityTypeEntry(entityTypeService.findByEntityType(context, "Project")), - EntityTypeMatcher.matchEntityTypeEntry(entityTypeService.findByEntityType(context, "OrgUnit")) + .matchEntityTypeEntry(entityTypeService.findByEntityType(context, "JournalIssue")), + EntityTypeMatcher + .matchEntityTypeEntry(entityTypeService.findByEntityType(context, "JournalVolume")), + EntityTypeMatcher.matchEntityTypeEntry(entityTypeService.findByEntityType(context, "OrgUnit")), + EntityTypeMatcher.matchEntityTypeEntry(entityTypeService.findByEntityType(context, "Person")) ))); getClient().perform(get("/api/core/entitytypes").param("size", "5").param("page", "1")) @@ -108,11 +108,12 @@ public class EntityTypeRestRepositoryIT extends AbstractEntityIntegrationTest { .andExpect(jsonPath("$._links.self.href", containsString("api/core/entitytypes"))) //We have 4 facets in the default configuration, they need to all be present in the embedded section .andExpect(jsonPath("$._embedded.entitytypes", containsInAnyOrder( - EntityTypeMatcher.matchEntityTypeEntry(entityTypeService.findByEntityType(context, "Journal")), + EntityTypeMatcher.matchEntityTypeEntry(entityTypeService.findByEntityType(context, "Project")), EntityTypeMatcher - .matchEntityTypeEntry(entityTypeService.findByEntityType(context, "JournalVolume")), + .matchEntityTypeEntry(entityTypeService.findByEntityType(context, "Publication")), EntityTypeMatcher - .matchEntityTypeEntry(entityTypeService.findByEntityType(context, "JournalIssue")) + .matchEntityTypeEntry(entityTypeService.findByEntityType(context, + Constants.ENTITY_TYPE_NONE)) ))); } @@ -137,10 +138,11 @@ public class EntityTypeRestRepositoryIT extends AbstractEntityIntegrationTest { .param("size", "3")) .andExpect(status().isOk()) .andExpect(jsonPath("$._embedded.entitytypes", containsInAnyOrder( - EntityTypeMatcher.matchEntityTypeEntry(entityTypeService.findByEntityType(context, "Publication")), - EntityTypeMatcher.matchEntityTypeEntry(entityTypeService.findByEntityType(context, "Person")), + EntityTypeMatcher.matchEntityTypeEntry(entityTypeService.findByEntityType(context, "Journal")), EntityTypeMatcher.matchEntityTypeEntry(entityTypeService.findByEntityType(context, - Constants.ENTITY_TYPE_NONE)) + "JournalIssue")), + EntityTypeMatcher.matchEntityTypeEntry(entityTypeService.findByEntityType(context, + "JournalVolume")) ))) .andExpect(jsonPath("$._links.first.href", Matchers.allOf( Matchers.containsString("/api/core/entitytypes?"), @@ -165,7 +167,7 @@ public class EntityTypeRestRepositoryIT extends AbstractEntityIntegrationTest { .andExpect(status().isOk()) .andExpect(jsonPath("$._embedded.entitytypes", containsInAnyOrder( EntityTypeMatcher.matchEntityTypeEntry(entityTypeService.findByEntityType(context, "OrgUnit")), - EntityTypeMatcher.matchEntityTypeEntry(entityTypeService.findByEntityType(context, "Journal")), + EntityTypeMatcher.matchEntityTypeEntry(entityTypeService.findByEntityType(context, "Person")), EntityTypeMatcher.matchEntityTypeEntry(entityTypeService.findByEntityType(context, "Project")) ))) .andExpect(jsonPath("$._links.first.href", Matchers.allOf( diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ItemRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ItemRestRepositoryIT.java index 372ab084dd..95ec537727 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ItemRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ItemRestRepositoryIT.java @@ -123,9 +123,11 @@ public class ItemRestRepositoryIT extends AbstractControllerIntegrationTest { Collection col2 = CollectionBuilder.createCollection(context, child1).withName("Collection 2").build(); List items = new ArrayList(); - // This comparator is used to sort our test Items by java.util.UUID (which sorts them based on the RFC - // and not based on String comparison, see also https://stackoverflow.com/a/51031298/3750035 ) - Comparator compareByUUID = Comparator.comparing(i -> i.getID()); + // Hibernate 5.x's org.hibernate.dialect.H2Dialect sorts UUIDs as if they are Strings. + // So, we must compare UUIDs as if they are strings. + // In Hibernate 6, the H2Dialect has been updated with native UUID type support, at which point + // we'd need to update the below comparator to compare them as java.util.UUID (which sorts based on RFC 4412). + Comparator compareByUUID = Comparator.comparing(i -> i.getID().toString()); //2. Three public items that are readable by Anonymous with different subjects Item publicItem1 = ItemBuilder.createItem(context, col1) @@ -204,9 +206,11 @@ public class ItemRestRepositoryIT extends AbstractControllerIntegrationTest { .build(); List items = new ArrayList(); - // This comparator is used to sort our test Items by java.util.UUID (which sorts them based on the RFC - // and not based on String comparison, see also https://stackoverflow.com/a/51031298/3750035 ) - Comparator compareByUUID = Comparator.comparing(i -> i.getID()); + // Hibernate 5.x's org.hibernate.dialect.H2Dialect sorts UUIDs as if they are Strings. + // So, we must compare UUIDs as if they are strings. + // In Hibernate 6, the H2Dialect has been updated with native UUID type support, at which point + // we'd need to update the below comparator to compare them as java.util.UUID (which sorts based on RFC 4412). + Comparator compareByUUID = Comparator.comparing(i -> i.getID().toString()); //2. Three public items that are readable by Anonymous with different subjects Item publicItem1 = ItemBuilder.createItem(context, col1) @@ -3213,16 +3217,23 @@ public class ItemRestRepositoryIT extends AbstractControllerIntegrationTest { Item item = ItemBuilder.createItem(context, collection).withTitle("Item").build(); - Bundle bundle0 = BundleBuilder.createBundle(context, item).withName("Bundle 0").build(); - Bundle bundle1 = BundleBuilder.createBundle(context, item).withName("Bundle 1").build(); - Bundle bundle2 = BundleBuilder.createBundle(context, item).withName("Bundle 2").build(); - Bundle bundle3 = BundleBuilder.createBundle(context, item).withName("Bundle 3").build(); - Bundle bundle4 = BundleBuilder.createBundle(context, item).withName("Bundle 4").build(); - Bundle bundle5 = BundleBuilder.createBundle(context, item).withName("Bundle 5").build(); - Bundle bundle6 = BundleBuilder.createBundle(context, item).withName("Bundle 6").build(); - Bundle bundle7 = BundleBuilder.createBundle(context, item).withName("Bundle 7").build(); - Bundle bundle8 = BundleBuilder.createBundle(context, item).withName("Bundle 8").build(); - Bundle bundle9 = BundleBuilder.createBundle(context, item).withName("Bundle 9").build(); + List bundles = new ArrayList(); + bundles.add(BundleBuilder.createBundle(context, item).withName("Bundle 0").build()); + bundles.add(BundleBuilder.createBundle(context, item).withName("Bundle 1").build()); + bundles.add(BundleBuilder.createBundle(context, item).withName("Bundle 2").build()); + bundles.add(BundleBuilder.createBundle(context, item).withName("Bundle 3").build()); + bundles.add(BundleBuilder.createBundle(context, item).withName("Bundle 4").build()); + bundles.add(BundleBuilder.createBundle(context, item).withName("Bundle 5").build()); + bundles.add(BundleBuilder.createBundle(context, item).withName("Bundle 6").build()); + bundles.add(BundleBuilder.createBundle(context, item).withName("Bundle 7").build()); + bundles.add(BundleBuilder.createBundle(context, item).withName("Bundle 8").build()); + bundles.add(BundleBuilder.createBundle(context, item).withName("Bundle 9").build()); + + // While in DSpace code, Bundles are *unordered*, in Hibernate v5 + H2 v2.x, they are returned sorted by UUID. + // So, we reorder this list of created Bundles by UUID to get their expected pagination ordering. + // NOTE: Once on Hibernate v6, this might need "toString()" removed as it may sort UUIDs based on RFC 4412. + Comparator compareByUUID = Comparator.comparing(b -> b.getID().toString()); + bundles.sort(compareByUUID); context.restoreAuthSystemState(); @@ -3232,11 +3243,16 @@ public class ItemRestRepositoryIT extends AbstractControllerIntegrationTest { .andExpect(status().isOk()) .andExpect(jsonPath("$", ItemMatcher.matchItemProperties(item))) .andExpect(jsonPath("$._embedded.bundles._embedded.bundles",Matchers.containsInAnyOrder( - BundleMatcher.matchProperties(bundle0.getName(), bundle0.getID(), bundle0.getHandle(), bundle0.getType()), - BundleMatcher.matchProperties(bundle1.getName(), bundle1.getID(), bundle1.getHandle(), bundle1.getType()), - BundleMatcher.matchProperties(bundle2.getName(), bundle2.getID(), bundle2.getHandle(), bundle2.getType()), - BundleMatcher.matchProperties(bundle3.getName(), bundle3.getID(), bundle3.getHandle(), bundle3.getType()), - BundleMatcher.matchProperties(bundle4.getName(), bundle4.getID(), bundle4.getHandle(), bundle4.getType()) + BundleMatcher.matchProperties(bundles.get(0).getName(), bundles.get(0).getID(), bundles.get(0).getHandle(), + bundles.get(0).getType()), + BundleMatcher.matchProperties(bundles.get(1).getName(), bundles.get(1).getID(), bundles.get(1).getHandle(), + bundles.get(1).getType()), + BundleMatcher.matchProperties(bundles.get(2).getName(), bundles.get(2).getID(), bundles.get(2).getHandle(), + bundles.get(2).getType()), + BundleMatcher.matchProperties(bundles.get(3).getName(), bundles.get(3).getID(), bundles.get(3).getHandle(), + bundles.get(3).getType()), + BundleMatcher.matchProperties(bundles.get(4).getName(), bundles.get(4).getID(), bundles.get(4).getHandle(), + bundles.get(4).getType()) ))) .andExpect(jsonPath("$._links.self.href", Matchers.containsString("/api/core/items/" + item.getID()))) .andExpect(jsonPath("$._embedded.bundles.page.size", is(5))) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/RelationshipTypeRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/RelationshipTypeRestRepositoryIT.java index a2ed672a61..3295e03593 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/RelationshipTypeRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/RelationshipTypeRestRepositoryIT.java @@ -22,13 +22,13 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. import java.sql.SQLException; import java.util.List; +import org.apache.commons.lang3.StringUtils; import org.dspace.app.rest.matcher.EntityTypeMatcher; import org.dspace.app.rest.matcher.RelationshipTypeMatcher; import org.dspace.app.rest.test.AbstractEntityIntegrationTest; import org.dspace.content.RelationshipType; import org.dspace.content.service.EntityTypeService; import org.dspace.content.service.RelationshipTypeService; -import org.h2.util.StringUtils; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/iiif/IIIFControllerIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/iiif/IIIFControllerIT.java index 71090cd70f..beafcb9622 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/iiif/IIIFControllerIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/iiif/IIIFControllerIT.java @@ -469,40 +469,39 @@ public class IIIFControllerIT extends AbstractControllerIntegrationTest { .withMimeType("image/tiff") .build(); } - context.restoreAuthSystemState(); - // expect structures elements with label and canvas id. + + // Expected structures elements based on the above test content + // NOTE: we cannot guarantee the order of Bundles in the Manifest, therefore this test has to simply check + // that each Bundle exists in the manifest with Canvases corresponding to each bitstream. getClient().perform(get("/iiif/" + publicItem1.getID() + "/manifest")) - .andExpect(status().isOk()) - .andExpect(jsonPath("$.@context", is("http://iiif.io/api/presentation/2/context.json"))) - .andExpect(jsonPath("$.sequences[0].canvases[0].@id", - Matchers.containsString("/iiif/" + publicItem1.getID() + "/canvas/c0"))) - .andExpect(jsonPath("$.sequences[0].canvases[0].label", is("Global 1"))) - .andExpect(jsonPath("$.sequences[0].canvases[0].width", is(2000))) - .andExpect(jsonPath("$.sequences[0].canvases[0].height", is(3000))) - .andExpect(jsonPath("$.sequences[0].canvases[1].label", is("Global 2"))) - .andExpect(jsonPath("$.sequences[0].canvases[2].label", is("Global 3"))) - .andExpect(jsonPath("$.structures[0].@id", - Matchers.endsWith("/iiif/" + publicItem1.getID() + "/manifest/range/r0"))) - .andExpect(jsonPath("$.structures[0].label", is("Table of Contents"))) - .andExpect(jsonPath("$.structures[0].viewingHint", is("top"))) - .andExpect(jsonPath("$.structures[0].ranges[0]", - Matchers.endsWith("/iiif/" + publicItem1.getID() + "/manifest/range/r0-0"))) - .andExpect(jsonPath("$.structures[0].ranges[1]", - Matchers.endsWith("/iiif/" + publicItem1.getID() + "/manifest/range/r0-1"))) - .andExpect(jsonPath("$.structures[1].@id", - Matchers.endsWith("/iiif/" + publicItem1.getID() + "/manifest/range/r0-0"))) - .andExpect(jsonPath("$.structures[1].label", is("ORIGINAL"))) - .andExpect(jsonPath("$.structures[1].canvases[0]", - Matchers.containsString("/iiif/" + publicItem1.getID() + "/canvas/c0"))) - .andExpect(jsonPath("$.structures[2].@id", - Matchers.endsWith("/iiif/" + publicItem1.getID() + "/manifest/range/r0-1"))) - .andExpect(jsonPath("$.structures[2].label", is("IIIF"))) - .andExpect(jsonPath("$.structures[2].canvases[0]", - Matchers.containsString("/iiif/" + publicItem1.getID() + "/canvas/c1"))) - .andExpect(jsonPath("$.structures[2].canvases[1]", - Matchers.containsString("/iiif/" + publicItem1.getID() + "/canvas/c2"))) - .andExpect(jsonPath("$.service").exists()); + .andExpect(status().isOk()) + .andExpect(jsonPath("$.@context", is("http://iiif.io/api/presentation/2/context.json"))) + // should contain 3 canvases, corresponding to each bitstream + .andExpect(jsonPath("$.sequences[0].canvases[*].label", + Matchers.contains("Global 1", "Global 2", "Global 3"))) + + // First structure should be a Table of Contents + .andExpect(jsonPath("$.structures[0].@id", + Matchers.endsWith("/iiif/" + publicItem1.getID() + "/manifest/range/r0"))) + .andExpect(jsonPath("$.structures[0].label", is("Table of Contents"))) + .andExpect(jsonPath("$.structures[0].viewingHint", is("top"))) + .andExpect(jsonPath("$.structures[0].ranges[0]", + Matchers.endsWith("/iiif/" + publicItem1.getID() + "/manifest/range/r0-0"))) + .andExpect(jsonPath("$.structures[0].ranges[1]", + Matchers.endsWith("/iiif/" + publicItem1.getID() + "/manifest/range/r0-1"))) + + // Should contain a structure with label=IIIF, corresponding to IIIF bundle + // It should have exactly 2 canvases (corresponding to 2 bitstreams) + .andExpect(jsonPath("$.structures[?(@.label=='IIIF')].canvases[0]").exists()) + .andExpect(jsonPath("$.structures[?(@.label=='IIIF')].canvases[1]").exists()) + .andExpect(jsonPath("$.structures[?(@.label=='IIIF')].canvases[2]").doesNotExist()) + + // Should contain a structure with label=ORIGINAL, corresponding to ORIGINAL bundle + // It should have exactly 1 canvas (corresponding to 1 bitstream) + .andExpect(jsonPath("$.structures[?(@.label=='ORIGINAL')].canvases[0]").exists()) + .andExpect(jsonPath("$.structures[?(@.label=='ORIGINAL')].canvases[1]").doesNotExist()) + .andExpect(jsonPath("$.service").exists()); } @Test diff --git a/dspace/config/dspace.cfg b/dspace/config/dspace.cfg index 6be09f25fc..0f607bebc5 100644 --- a/dspace/config/dspace.cfg +++ b/dspace/config/dspace.cfg @@ -80,9 +80,9 @@ db.url = jdbc:postgresql://localhost:5432/dspace db.driver = org.postgresql.Driver # Database Dialect (for Hibernate) -# * For Postgres: org.dspace.storage.rdbms.hibernate.postgres.DSpacePostgreSQL82Dialect +# * For Postgres: org.hibernate.dialect.PostgreSQL94Dialect # * For Oracle: org.hibernate.dialect.Oracle10gDialect -db.dialect = org.dspace.storage.rdbms.hibernate.postgres.DSpacePostgreSQL82Dialect +db.dialect = org.hibernate.dialect.PostgreSQL94Dialect # Database username and password db.username = dspace @@ -425,11 +425,7 @@ useProxies = true # perform automated format conversions #Names of the enabled MediaFilter or FormatFilter plugins -filter.plugins = PDF Text Extractor -filter.plugins = HTML Text Extractor -filter.plugins = Word Text Extractor -filter.plugins = Excel Text Extractor -filter.plugins = PowerPoint Text Extractor +filter.plugins = Text Extractor filter.plugins = JPEG Thumbnail filter.plugins = PDFBox JPEG Thumbnail @@ -443,11 +439,7 @@ filter.plugins = PDFBox JPEG Thumbnail # ImageMagick Image Thumbnail, ImageMagick PDF Thumbnail, \ #Assign 'human-understandable' names to each filter -plugin.named.org.dspace.app.mediafilter.FormatFilter = org.dspace.app.mediafilter.PDFFilter = PDF Text Extractor -plugin.named.org.dspace.app.mediafilter.FormatFilter = org.dspace.app.mediafilter.HTMLFilter = HTML Text Extractor -plugin.named.org.dspace.app.mediafilter.FormatFilter = org.dspace.app.mediafilter.PoiWordFilter = Word Text Extractor -plugin.named.org.dspace.app.mediafilter.FormatFilter = org.dspace.app.mediafilter.ExcelFilter = Excel Text Extractor -plugin.named.org.dspace.app.mediafilter.FormatFilter = org.dspace.app.mediafilter.PowerPointFilter = PowerPoint Text Extractor +plugin.named.org.dspace.app.mediafilter.FormatFilter = org.dspace.app.mediafilter.TikaTextExtractionFilter = Text Extractor plugin.named.org.dspace.app.mediafilter.FormatFilter = org.dspace.app.mediafilter.JPEGFilter = JPEG Thumbnail plugin.named.org.dspace.app.mediafilter.FormatFilter = org.dspace.app.mediafilter.BrandedPreviewJPEGFilter = Branded Preview JPEG plugin.named.org.dspace.app.mediafilter.FormatFilter = org.dspace.app.mediafilter.PDFBoxThumbnail = PDFBox JPEG Thumbnail @@ -455,15 +447,27 @@ plugin.named.org.dspace.app.mediafilter.FormatFilter = org.dspace.app.mediafilte plugin.named.org.dspace.app.mediafilter.FormatFilter = org.dspace.app.mediafilter.ImageMagickPdfThumbnailFilter = ImageMagick PDF Thumbnail #Configure each filter's input format(s) -filter.org.dspace.app.mediafilter.PDFFilter.inputFormats = Adobe PDF -filter.org.dspace.app.mediafilter.HTMLFilter.inputFormats = HTML, Text -filter.org.dspace.app.mediafilter.PoiWordFilter.inputFormats = Microsoft Word, Microsoft Word XML -filter.org.dspace.app.mediafilter.PowerPointFilter.inputFormats = Microsoft Powerpoint, Microsoft Powerpoint XML +# NOTE: The TikaTextExtractionFilter can support any file formats that are supported by Apache Tika. So, you can easily +# add additional formats to your DSpace Bitstream Format registry and list them here. The current list of Tika supported +# formats is available at: https://tika.apache.org/2.3.0/formats.html +filter.org.dspace.app.mediafilter.TikaTextExtractionFilter.inputFormats = Adobe PDF +filter.org.dspace.app.mediafilter.TikaTextExtractionFilter.inputFormats = CSV +filter.org.dspace.app.mediafilter.TikaTextExtractionFilter.inputFormats = HTML +filter.org.dspace.app.mediafilter.TikaTextExtractionFilter.inputFormats = Microsoft Excel +filter.org.dspace.app.mediafilter.TikaTextExtractionFilter.inputFormats = Microsoft Excel XML +filter.org.dspace.app.mediafilter.TikaTextExtractionFilter.inputFormats = Microsoft Powerpoint +filter.org.dspace.app.mediafilter.TikaTextExtractionFilter.inputFormats = Microsoft Powerpoint XML +filter.org.dspace.app.mediafilter.TikaTextExtractionFilter.inputFormats = Microsoft Word +filter.org.dspace.app.mediafilter.TikaTextExtractionFilter.inputFormats = Microsoft Word XML +filter.org.dspace.app.mediafilter.TikaTextExtractionFilter.inputFormats = OpenDocument Presentation +filter.org.dspace.app.mediafilter.TikaTextExtractionFilter.inputFormats = OpenDocument Spreadsheet +filter.org.dspace.app.mediafilter.TikaTextExtractionFilter.inputFormats = OpenDocument Text +filter.org.dspace.app.mediafilter.TikaTextExtractionFilter.inputFormats = RTF +filter.org.dspace.app.mediafilter.TikaTextExtractionFilter.inputFormats = Text filter.org.dspace.app.mediafilter.JPEGFilter.inputFormats = BMP, GIF, JPEG, image/png filter.org.dspace.app.mediafilter.BrandedPreviewJPEGFilter.inputFormats = BMP, GIF, JPEG, image/png filter.org.dspace.app.mediafilter.ImageMagickImageThumbnailFilter.inputFormats = BMP, GIF, image/png, JPG, TIFF, JPEG, JPEG 2000 filter.org.dspace.app.mediafilter.ImageMagickPdfThumbnailFilter.inputFormats = Adobe PDF -filter.org.dspace.app.mediafilter.ExcelFilter.inputFormats = Microsoft Excel, Microsoft Excel XML filter.org.dspace.app.mediafilter.PDFBoxThumbnail.inputFormats = Adobe PDF #Publicly accessible thumbnails of restricted content. @@ -471,15 +475,22 @@ filter.org.dspace.app.mediafilter.PDFBoxThumbnail.inputFormats = Adobe PDF #Any media filters not listed will instead inherit the permissions of the parent bitstream #filter.org.dspace.app.mediafilter.publicPermission = JPEGFilter -#Custom settings for PDFFilter -# If true, all PDF extractions are written to temp files as they are indexed...this -# is slower, but helps ensure that PDFBox software DSpace uses doesn't eat up -# all your memory -#pdffilter.largepdfs = true -# If true, PDFs which still result in an Out of Memory error from PDFBox -# are skipped over...these problematic PDFs will never be indexed until -# memory usage can be decreased in the PDFBox software -#pdffilter.skiponmemoryexception = true +# Custom settings for Text Extractor +# +# Maximum number of characters to be extracted for full text indexing +# Default is 100,000 characters. Set to -1 for no maximum. +# Larger values (or -1) are more likely to hit OutOfMemoryException errors when extracting text from large files. +# In those scenarios, consider instead setting "textextractor.use-temp-file = true" below. +# Changing this value only impacts future content added to DSpace. To modify existing content you must re-extract all +# text ("filter-media -f" ) and then reindex your site ("index-discovery -b"). +#textextractor.max-chars = 100000 + +# If true, all text extractions are written to temp files as they are indexed. +# This will be slower, but helps ensure that text extraction doesn't eat up all your memory. +# Setting this to true will also cause "textextractor.max-chars" to be ignored, and all text will be extracted. +# Changing this value only impacts future content added to DSpace. To modify existing content you must re-extract all +# text ("filter-media -f" ) and then reindex your site ("index-discovery -b"). +#textextractor.use-temp-file = false # Custom settigns for ImageMagick Thumbnail Filters # ImageMagick and GhostScript must be installed on the server, set the path to ImageMagick and GhostScript executable @@ -1425,8 +1436,7 @@ plugin.selfnamed.org.dspace.content.authority.ChoiceAuthority = \ authority.minconfidence = ambiguous # Configuration settings for ORCID based authority control. -# Uncomment the lines below to enable configuration. -#solr.authority.server=${solr.server}/${solr.multicorePrefix}authority +# Uncomment the lines below to enable configuration #choices.plugin.dc.contributor.author = SolrAuthorAuthority #choices.presentation.dc.contributor.author = authorLookup #authority.controlled.dc.contributor.author = true diff --git a/dspace/config/hibernate-ehcache-config.xml b/dspace/config/hibernate-ehcache-config.xml index 0900ce0b50..e2edf67b60 100644 --- a/dspace/config/hibernate-ehcache-config.xml +++ b/dspace/config/hibernate-ehcache-config.xml @@ -31,20 +31,22 @@ - - + + 6000 - - + + 600 diff --git a/dspace/config/local.cfg.EXAMPLE b/dspace/config/local.cfg.EXAMPLE index 167d2d45da..67c03808c4 100644 --- a/dspace/config/local.cfg.EXAMPLE +++ b/dspace/config/local.cfg.EXAMPLE @@ -80,9 +80,9 @@ db.url = jdbc:postgresql://localhost:5432/dspace db.driver = org.postgresql.Driver # Database Dialect (for Hibernate) -# * For Postgres: org.dspace.storage.rdbms.hibernate.postgres.DSpacePostgreSQL82Dialect +# * For Postgres: org.hibernate.dialect.PostgreSQL94Dialect # * For Oracle: org.hibernate.dialect.Oracle10gDialect -db.dialect = org.dspace.storage.rdbms.hibernate.postgres.DSpacePostgreSQL82Dialect +db.dialect = org.hibernate.dialect.PostgreSQL94Dialect # Database username and password db.username = dspace diff --git a/dspace/config/modules/solrauthority.cfg b/dspace/config/modules/solrauthority.cfg index 9ed1855d44..afa1bdd22a 100644 --- a/dspace/config/modules/solrauthority.cfg +++ b/dspace/config/modules/solrauthority.cfg @@ -4,6 +4,10 @@ # These configs are only used by the SOLR authority index # #---------------------------------------------------------------# +# Solr Authority index location +# Default is ${solr.server}/authority, unless solr.multicorePrefix is specified +solr.authority.server=${solr.server}/${solr.multicorePrefix}authority + # Update item metadata displayed values (not the authority keys) # with the lasted cached versions when running the UpdateAuthorities index script #solrauthority.auto-update-items=false diff --git a/dspace/config/registries/bitstream-formats.xml b/dspace/config/registries/bitstream-formats.xml index 0b55bcea47..586931b376 100644 --- a/dspace/config/registries/bitstream-formats.xml +++ b/dspace/config/registries/bitstream-formats.xml @@ -106,6 +106,15 @@ css + + text/csv + CSV + Comma-Separated Values + 1 + false + csv + + application/msword Microsoft Word diff --git a/dspace/modules/additions/pom.xml b/dspace/modules/additions/pom.xml index 9df5c8ef02..569ca8ce25 100644 --- a/dspace/modules/additions/pom.xml +++ b/dspace/modules/additions/pom.xml @@ -254,30 +254,54 @@ - - org.apache.lucene - lucene-core - - - org.apache.lucene - lucene-analyzers-icu - test - - - org.apache.lucene - lucene-analyzers-smartcn - test - - - org.apache.lucene - lucene-analyzers-stempel - test - - - org.apache.xmlbeans - xmlbeans - - + + org.apache.solr + solr-solrj + ${solr.client.version} + + + + + org.apache.solr + solr-core + ${solr.client.version} + test + + + + org.apache.commons + commons-text + + + + org.eclipse.jetty + jetty-http + + + org.eclipse.jetty + jetty-io + + + org.eclipse.jetty + jetty-util + + + + + org.apache.lucene + lucene-analyzers-icu + test + + + org.apache.lucene + lucene-analyzers-smartcn + test + + + org.apache.lucene + lucene-analyzers-stempel + test + junit junit diff --git a/dspace/modules/server/pom.xml b/dspace/modules/server/pom.xml index c789cf5054..dd854c4215 100644 --- a/dspace/modules/server/pom.xml +++ b/dspace/modules/server/pom.xml @@ -278,6 +278,11 @@ just adding new jar in the classloader dspace-server-webapp war + + org.apache.solr + solr-solrj + ${solr.client.version} + @@ -324,9 +329,12 @@ just adding new jar in the classloader mockito-inline test + + org.apache.solr - solr-cell + solr-core + ${solr.client.version} test diff --git a/dspace/solr/search/conf/solrconfig.xml b/dspace/solr/search/conf/solrconfig.xml index 204c1a056e..8d45105df5 100644 --- a/dspace/solr/search/conf/solrconfig.xml +++ b/dspace/solr/search/conf/solrconfig.xml @@ -30,9 +30,6 @@ regex='icu4j-.*\.jar'/> - - ${solr.data.dir:} diff --git a/dspace/src/main/docker-compose/cli.assetstore.yml b/dspace/src/main/docker-compose/cli.assetstore.yml index 7773facae6..d6a6a4395f 100644 --- a/dspace/src/main/docker-compose/cli.assetstore.yml +++ b/dspace/src/main/docker-compose/cli.assetstore.yml @@ -24,6 +24,6 @@ services: tar xvfz /tmp/assetstore.tar.gz fi - /dspace/bin/dspace index-discovery + /dspace/bin/dspace index-discovery -b /dspace/bin/dspace oai import /dspace/bin/dspace oai clean-cache diff --git a/dspace/src/main/docker-compose/db.entities.yml b/dspace/src/main/docker-compose/db.entities.yml index 762519f508..8d86f7bb83 100644 --- a/dspace/src/main/docker-compose/db.entities.yml +++ b/dspace/src/main/docker-compose/db.entities.yml @@ -13,7 +13,7 @@ services: image: dspace/dspace-postgres-pgcrypto:loadsql environment: # This SQL is available from https://github.com/DSpace-Labs/AIP-Files/releases/tag/demo-entities-data - - LOADSQL=https://github.com/DSpace-Labs/AIP-Files/releases/download/demo-entities-data/dspace7-entities-2021-04-14.sql + - LOADSQL=https://github.com/DSpace-Labs/AIP-Files/releases/download/demo-entities-data/dspace7-entities-data.sql dspace: ### OVERRIDE default 'entrypoint' in 'docker-compose.yml #### # Ensure that the database is ready BEFORE starting tomcat diff --git a/pom.xml b/pom.xml index e03dcb3b07..734a4fd3d6 100644 --- a/pom.xml +++ b/pom.xml @@ -22,10 +22,10 @@ 5.2.5.RELEASE 2.2.6.RELEASE 5.2.2.RELEASE - 5.4.10.Final - 6.0.18.Final + 5.6.5.Final + 6.0.23.Final 42.3.3 - 8.8.1 + 8.11.1 3.4.0 2.10.0 @@ -36,12 +36,14 @@ 2.3.1 1.1.0 - 9.4.41.v20210516 + 9.4.44.v20210927 2.17.1 2.0.24 - 3.17 1.18.0 1.7.25 + 2.3.0 + + 1.70 @@ -57,7 +59,7 @@ https://jena.apache.org/documentation/migrate_jena2_jena3.html --> 2.13.0 - 2.30.1 + 2.35 UTF-8 @@ -1106,28 +1108,13 @@ org.postgresql postgresql ${postgresql.driver.version} - - - - - org.apache.zookeeper - zookeeper - 3.4.14 - - - - - org.apache.james - apache-mime4j-core - 0.8.3 - - - - - org.ow2.asm - asm - 8.0.1 + + + + org.checkerframework + checker-qual + + @@ -1161,6 +1148,13 @@ ${hibernate-validator.version} + + + org.jboss.logging + jboss-logging + 3.4.3.Final + + com.rometools @@ -1279,7 +1273,7 @@ org.apache.solr - solr-cell + solr-solrj ${solr.client.version} @@ -1287,35 +1281,43 @@ lucene-core ${solr.client.version} - + + org.apache.tika - tika-parsers - 1.24.1 - - - - com.google.protobuf - protobuf-java - - - net.sf.ehcache - ehcache-core - - - jakarta.ws.rs - jakarta.ws.rs-api - - - javax.xml.soap - javax.xml.soap-api - - - org.jvnet.staxex - stax-ex - - + tika-core + ${tika.version} + + org.apache.tika + tika-parsers-standard-package + ${tika.version} + + + + org.bouncycastle + bcpkix-jdk15on + ${bouncycastle.version} + + + + org.bouncycastle + bcprov-jdk15on + ${bouncycastle.version} + + + + org.apache.james + apache-mime4j-core + 0.8.4 + + + + org.ow2.asm + asm + 8.0.1 + + @@ -1353,7 +1355,7 @@ org.slf4j slf4j-log4j12 - + org.apache.commons commons-csv @@ -1549,45 +1551,6 @@ fontbox ${pdfbox-version} - - org.apache.poi - poi - ${poi-version} - - - org.apache.poi - poi-scratchpad - ${poi-version} - - - org.apache.poi - poi-ooxml - ${poi-version} - - - - com.github.virtuald - curvesapi - - - - - org.apache.poi - poi-ooxml-schemas - ${poi-version} - - - - org.apache.xmlbeans - xmlbeans - - - - - org.apache.xmlbeans - xmlbeans - 3.1.0 - xalan xalan @@ -1691,7 +1654,7 @@ com.h2database h2 - 1.4.187 + 2.1.210 test @@ -1775,7 +1738,14 @@ com.google.guava guava - 30.0-jre + 30.1.1-jre + + + + org.checkerframework + checker-qual + + xom