From 64046093d2cb45ec2bcf6f3396953c9d2f9f9315 Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Mon, 22 Jun 2020 11:46:53 +0200 Subject: [PATCH 01/16] Bibtex file import --- ...BibtexImportMetadataSourceServiceImpl.java | 107 ++++++++++++++ .../FileMultipleOccurencesException.java | 29 ++++ .../exception/FileSourceException.java | 28 ++++ .../AbstractMetadataFieldMapping.java | 3 - .../SimpleMetadataContributor.java | 97 +++++++++++++ ...PubmedImportMetadataSourceServiceImpl.java | 5 +- .../AbstractImportMetadataSourceService.java | 3 +- .../external/service/ImportService.java | 83 ++++++++--- .../AbstractPlainMetadataSource.java | 86 +++++++++++ .../service/components/FileSource.java | 46 ++++++ .../service/components/MetadataSource.java | 85 +---------- .../service/components/QuerySource.java | 106 ++++++++++++++ .../dto/PlainMetadataKeyValueItem.java | 37 +++++ .../dto/PlainMetadataSourceDto.java | 32 +++++ .../spring-dspace-addon-import-services.xml | 22 ++- .../WorkspaceItemRestRepository.java | 136 ++++++------------ .../dspace/app/rest/bibtex-test-3-entries.bib | 14 ++ .../org/dspace/app/rest/bibtex-test.bib | 12 +- .../config/spring/api/bibtex-integration.xml | 74 ++++++++++ .../config/spring/api/external-services.xml | 7 +- 20 files changed, 801 insertions(+), 211 deletions(-) create mode 100644 dspace-api/src/main/java/org/dspace/importer/external/bibtex/service/BibtexImportMetadataSourceServiceImpl.java create mode 100644 dspace-api/src/main/java/org/dspace/importer/external/exception/FileMultipleOccurencesException.java create mode 100644 dspace-api/src/main/java/org/dspace/importer/external/exception/FileSourceException.java create mode 100644 dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/SimpleMetadataContributor.java create mode 100644 dspace-api/src/main/java/org/dspace/importer/external/service/components/AbstractPlainMetadataSource.java create mode 100644 dspace-api/src/main/java/org/dspace/importer/external/service/components/FileSource.java create mode 100644 dspace-api/src/main/java/org/dspace/importer/external/service/components/QuerySource.java create mode 100644 dspace-api/src/main/java/org/dspace/importer/external/service/components/dto/PlainMetadataKeyValueItem.java create mode 100644 dspace-api/src/main/java/org/dspace/importer/external/service/components/dto/PlainMetadataSourceDto.java create mode 100644 dspace-server-webapp/src/test/resources/org/dspace/app/rest/bibtex-test-3-entries.bib create mode 100644 dspace/config/spring/api/bibtex-integration.xml diff --git a/dspace-api/src/main/java/org/dspace/importer/external/bibtex/service/BibtexImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/bibtex/service/BibtexImportMetadataSourceServiceImpl.java new file mode 100644 index 0000000000..7468d601f5 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/bibtex/service/BibtexImportMetadataSourceServiceImpl.java @@ -0,0 +1,107 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ + +package org.dspace.importer.external.bibtex.service; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import javax.annotation.Resource; + +import org.dspace.importer.external.exception.FileSourceException; +import org.dspace.importer.external.service.components.AbstractPlainMetadataSource; +import org.dspace.importer.external.service.components.dto.PlainMetadataKeyValueItem; +import org.dspace.importer.external.service.components.dto.PlainMetadataSourceDto; +import org.jbibtex.BibTeXDatabase; +import org.jbibtex.BibTeXEntry; +import org.jbibtex.BibTeXParser; +import org.jbibtex.Key; +import org.jbibtex.ParseException; +import org.jbibtex.Value; + +/** + * Implements a metadata importer for BibTeX files + * + * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it) + */ +public class BibtexImportMetadataSourceServiceImpl extends AbstractPlainMetadataSource { + + + /** + * The string that identifies this import implementation as + * MetadataSource implementation + * + * @return the identifying uri + */ + @Override + public String getImportSource() { + return "BibTeXMetadataSource"; + } + + @Override + protected List readData (InputStream + inputStream) throws FileSourceException { + List list = new ArrayList<>(); + BibTeXDatabase database; + try { + database = parseBibTex(inputStream); + } catch (IOException | ParseException e) { + throw new FileSourceException("Unable to parse file with BibTeX parser"); + } + if (database == null || database.getEntries() == null) { + throw new FileSourceException("File results in an empty list of metadata"); + } + if (database.getEntries() != null) { + for (Entry entry : database.getEntries().entrySet()) { + PlainMetadataSourceDto item = new PlainMetadataSourceDto(); + List keyValues = new ArrayList<>(); + item.setMetadata(keyValues); + PlainMetadataKeyValueItem keyValueItem = new PlainMetadataKeyValueItem(); + keyValueItem.setKey(entry.getValue().getType().getValue()); + keyValueItem.setValue(entry.getKey().getValue()); + keyValues.add(keyValueItem); + if (entry.getValue().getFields() != null) { + for (Entry subentry : entry.getValue().getFields().entrySet()) { + PlainMetadataKeyValueItem innerItem = new PlainMetadataKeyValueItem(); + innerItem.setKey(subentry.getKey().getValue()); + innerItem.setValue(subentry.getValue().toUserString()); + keyValues.add(innerItem); + } + } + list.add(item); + } + } + return list; + } + + private BibTeXDatabase parseBibTex(InputStream inputStream) throws IOException, ParseException { + Reader reader = new InputStreamReader(inputStream); + BibTeXParser bibtexParser = new BibTeXParser(); + return bibtexParser.parse(reader); + } + + + /** + * Retrieve the MetadataFieldMapping containing the mapping between RecordType + * (in this case PlainMetadataSourceDto.class) and Metadata + * + * @return The configured MetadataFieldMapping + */ + @Override + @SuppressWarnings("unchecked") + @Resource(name = "bibtexMetadataFieldMap") + public void setMetadataFieldMap(@SuppressWarnings("rawtypes") Map metadataFieldMap) { + super.setMetadataFieldMap(metadataFieldMap); + } + +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/exception/FileMultipleOccurencesException.java b/dspace-api/src/main/java/org/dspace/importer/external/exception/FileMultipleOccurencesException.java new file mode 100644 index 0000000000..d09889a7ff --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/exception/FileMultipleOccurencesException.java @@ -0,0 +1,29 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ + +package org.dspace.importer.external.exception; + +/** + * This exception could be throws when more than one element is found + * in a method that works on one only. + * + * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it) + */ + +public class FileMultipleOccurencesException extends Exception { + + private static final long serialVersionUID = 1222409723339501937L; + + public FileMultipleOccurencesException(String message, Throwable cause) { + super(message, cause); + } + + public FileMultipleOccurencesException(String message) { + super(message); + } +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/exception/FileSourceException.java b/dspace-api/src/main/java/org/dspace/importer/external/exception/FileSourceException.java new file mode 100644 index 0000000000..c41ce94151 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/exception/FileSourceException.java @@ -0,0 +1,28 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ + +package org.dspace.importer.external.exception; + +/** + * Represents a problem with the File content: e.g. null input stream, invalid content, ... + * + * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it) + */ + +public class FileSourceException extends Exception { + + private static final long serialVersionUID = 6895579588455260182L; + + public FileSourceException(String message, Throwable cause) { + super(message, cause); + } + + public FileSourceException(String message) { + super(message); + } +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/AbstractMetadataFieldMapping.java b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/AbstractMetadataFieldMapping.java index 3ce45d6048..aed2f0e084 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/AbstractMetadataFieldMapping.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/AbstractMetadataFieldMapping.java @@ -117,16 +117,13 @@ public abstract class AbstractMetadataFieldMapping public Collection resultToDCValueMapping(RecordType record) { List values = new LinkedList(); - for (MetadataContributor query : getMetadataFieldMap().values()) { try { values.addAll(query.contributeMetadata(record)); } catch (Exception e) { log.error("Error", e); } - } return values; - } } diff --git a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/SimpleMetadataContributor.java b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/SimpleMetadataContributor.java new file mode 100644 index 0000000000..8fcbfdb8b8 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/SimpleMetadataContributor.java @@ -0,0 +1,97 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ + +package org.dspace.importer.external.metadatamapping.contributor; + +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; + +import org.dspace.importer.external.metadatamapping.MetadataFieldConfig; +import org.dspace.importer.external.metadatamapping.MetadataFieldMapping; +import org.dspace.importer.external.metadatamapping.MetadatumDTO; +import org.dspace.importer.external.service.components.dto.PlainMetadataKeyValueItem; +import org.dspace.importer.external.service.components.dto.PlainMetadataSourceDto; + +/** + * Metadata contributor that takes an PlainMetadataSourceDto instance and turns it into a metadatum + * + * @author Roeland Dillen (roeland at atmire dot com) + */ +public class SimpleMetadataContributor implements MetadataContributor { + + private MetadataFieldConfig field; + + private String key; + + private MetadataFieldMapping> metadataFieldMapping; + + public SimpleMetadataContributor(MetadataFieldConfig field, String key) { + this.field = field; + this.key = key; + } + + public SimpleMetadataContributor() { } + + /** + * Set the metadataFieldMapping of this SimpleMetadataContributor + * + * @param metadataFieldMapping the new mapping. + */ + @Override + public void setMetadataFieldMapping( + MetadataFieldMapping> metadataFieldMapping) { + this.metadataFieldMapping = metadataFieldMapping; + } + + /** + * Retrieve the metadata associated with the given object. + * It match the key found in PlainMetadataSourceDto instance with the key passed to constructor. + * In case of success, new metadatum is constructer (using field elements and PlainMetadataSourceDto value) + * and added to the list. + * + * @param t A class to retrieve metadata and key to match from. + * @return a collection of import records. Only the identifier of the found records may be put in the record. + */ + @Override + public Collection contributeMetadata(PlainMetadataSourceDto t) { + List values = new LinkedList<>(); + try { + for (PlainMetadataKeyValueItem metadatum : t.getMetadata()) { + if (metadatum.getKey().equals(key)) { + MetadatumDTO dcValue = new MetadatumDTO(); + dcValue.setValue(metadatum.getValue()); + dcValue.setElement(field.getElement()); + dcValue.setQualifier(field.getQualifier()); + dcValue.setSchema(field.getSchema()); + values.add(dcValue); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return values; + } + + /* + * Setter to inject field item + */ + public void setField(MetadataFieldConfig field) { + this.field = field; + } + + /* + * Setter to inject key value + */ + public void setKey(String key) { + this.key = key; + } + +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/pubmed/service/PubmedImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/pubmed/service/PubmedImportMetadataSourceServiceImpl.java index bcb5034301..02d205914a 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/pubmed/service/PubmedImportMetadataSourceServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/pubmed/service/PubmedImportMetadataSourceServiceImpl.java @@ -29,6 +29,7 @@ import org.dspace.importer.external.datamodel.ImportRecord; import org.dspace.importer.external.datamodel.Query; import org.dspace.importer.external.exception.MetadataSourceException; import org.dspace.importer.external.service.AbstractImportMetadataSourceService; +import org.dspace.importer.external.service.components.QuerySource; import org.jaxen.JaxenException; /** @@ -36,7 +37,9 @@ import org.jaxen.JaxenException; * * @author Roeland Dillen (roeland at atmire dot com) */ -public class PubmedImportMetadataSourceServiceImpl extends AbstractImportMetadataSourceService { +public class PubmedImportMetadataSourceServiceImpl extends AbstractImportMetadataSourceService + implements QuerySource { + private String baseAddress; private WebTarget pubmedWebTarget; diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/AbstractImportMetadataSourceService.java b/dspace-api/src/main/java/org/dspace/importer/external/service/AbstractImportMetadataSourceService.java index a803958a9d..f904ce613c 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/service/AbstractImportMetadataSourceService.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/AbstractImportMetadataSourceService.java @@ -28,8 +28,7 @@ import org.springframework.beans.factory.annotation.Required; * * @author Roeland Dillen (roeland at atmire dot com) */ -public abstract class AbstractImportMetadataSourceService extends AbstractRemoteMetadataSource - implements MetadataSource { +public abstract class AbstractImportMetadataSourceService extends AbstractRemoteMetadataSource { private GenerateQueryService generateQueryForItem = null; private MetadataFieldMapping> metadataFieldMapping; diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java b/dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java index 87c2bd0029..fe9f473137 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java @@ -8,6 +8,7 @@ package org.dspace.importer.external.service; +import java.io.InputStream; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -19,11 +20,16 @@ import org.apache.logging.log4j.Logger; import org.dspace.content.Item; import org.dspace.importer.external.datamodel.ImportRecord; import org.dspace.importer.external.datamodel.Query; +import org.dspace.importer.external.exception.FileMultipleOccurencesException; +import org.dspace.importer.external.exception.FileSourceException; import org.dspace.importer.external.exception.MetadataSourceException; +import org.dspace.importer.external.service.components.AbstractPlainMetadataSource; import org.dspace.importer.external.service.components.Destroyable; import org.dspace.importer.external.service.components.MetadataSource; +import org.dspace.importer.external.service.components.QuerySource; import org.springframework.beans.factory.annotation.Autowired; + /** * Main entry point for the import framework. * Instead of calling the different importer implementations, the ImportService should be called instead. @@ -32,8 +38,10 @@ import org.springframework.beans.factory.annotation.Autowired; * importer implementation you want to use. * * @author Roeland Dillen (roeland at atmire dot com) + * @author Pasquale Cavallo (pasquale.cavallo@4science.it) */ public class ImportService implements Destroyable { + private HashMap importSources = new HashMap<>(); Logger log = org.apache.logging.log4j.LogManager.getLogger(ImportService.class); @@ -101,11 +109,11 @@ public class ImportService implements Destroyable { public Collection findMatchingRecords(String uri, Item item) throws MetadataSourceException { try { List recordList = new LinkedList(); - for (MetadataSource metadataSource : matchingImports(uri)) { - recordList.addAll(metadataSource.findMatchingRecords(item)); + if (metadataSource instanceof QuerySource) { + recordList.addAll(((QuerySource)metadataSource).findMatchingRecords(item)); + } } - return recordList; } catch (Exception e) { throw new MetadataSourceException(e); @@ -125,9 +133,10 @@ public class ImportService implements Destroyable { try { List recordList = new LinkedList(); for (MetadataSource metadataSource : matchingImports(uri)) { - recordList.addAll(metadataSource.findMatchingRecords(query)); + if (metadataSource instanceof QuerySource) { + recordList.addAll(((QuerySource)metadataSource).findMatchingRecords(query)); + } } - return recordList; } catch (Exception e) { throw new MetadataSourceException(e); @@ -145,8 +154,10 @@ public class ImportService implements Destroyable { public int getNbRecords(String uri, String query) throws MetadataSourceException { try { int total = 0; - for (MetadataSource MetadataSource : matchingImports(uri)) { - total += MetadataSource.getNbRecords(query); + for (MetadataSource metadataSource : matchingImports(uri)) { + if (metadataSource instanceof QuerySource) { + total += ((QuerySource)metadataSource).getNbRecords(query); + } } return total; } catch (Exception e) { @@ -165,8 +176,10 @@ public class ImportService implements Destroyable { public int getNbRecords(String uri, Query query) throws MetadataSourceException { try { int total = 0; - for (MetadataSource MetadataSource : matchingImports(uri)) { - total += MetadataSource.getNbRecords(query); + for (MetadataSource metadataSource : matchingImports(uri)) { + if (metadataSource instanceof QuerySource) { + total += ((QuerySource)metadataSource).getNbRecords(query); + } } return total; } catch (Exception e) { @@ -189,7 +202,9 @@ public class ImportService implements Destroyable { try { List recordList = new LinkedList<>(); for (MetadataSource metadataSource : matchingImports(uri)) { - recordList.addAll(metadataSource.getRecords(query, start, count)); + if (metadataSource instanceof QuerySource) { + recordList.addAll(((QuerySource)metadataSource).getRecords(query, start, count)); + } } return recordList; } catch (Exception e) { @@ -209,7 +224,9 @@ public class ImportService implements Destroyable { try { List recordList = new LinkedList<>(); for (MetadataSource metadataSource : matchingImports(uri)) { - recordList.addAll(metadataSource.getRecords(query)); + if (metadataSource instanceof QuerySource) { + recordList.addAll(((QuerySource)metadataSource).getRecords(query)); + } } return recordList; } catch (Exception e) { @@ -229,10 +246,12 @@ public class ImportService implements Destroyable { public ImportRecord getRecord(String uri, String id) throws MetadataSourceException { try { for (MetadataSource metadataSource : matchingImports(uri)) { - if (metadataSource.getRecord(id) != null) { - return metadataSource.getRecord(id); + if (metadataSource instanceof QuerySource) { + QuerySource querySource = (QuerySource)metadataSource; + if (querySource.getRecord(id) != null) { + return querySource.getRecord(id); + } } - } return null; } catch (Exception e) { @@ -252,10 +271,12 @@ public class ImportService implements Destroyable { public ImportRecord getRecord(String uri, Query query) throws MetadataSourceException { try { for (MetadataSource metadataSource : matchingImports(uri)) { - if (metadataSource.getRecord(query) != null) { - return metadataSource.getRecord(query); + if (metadataSource instanceof QuerySource) { + QuerySource querySource = (QuerySource)metadataSource; + if (querySource.getRecord(query) != null) { + return querySource.getRecord(query); + } } - } return null; } catch (Exception e) { @@ -272,6 +293,34 @@ public class ImportService implements Destroyable { return importSources.keySet(); } + /* + * Get a collection of record from InputStream, + * The first match will be return. + * This method doesn't close the InputStream. + * + * @param fileInputStream the input stream to the resource + * @return a single record contains the metadatum + * @throws FileMultipleOccurencesException if more than one entry is found + */ + public ImportRecord getRecord(InputStream fileInputStream) throws FileMultipleOccurencesException { + ImportRecord importRecords = null; + for (MetadataSource metadataSource : importSources.values()) { + if (metadataSource instanceof AbstractPlainMetadataSource) { + AbstractPlainMetadataSource fileSource = (AbstractPlainMetadataSource)metadataSource; + try { + importRecords = fileSource.getRecord(fileInputStream); + break; + } catch (FileSourceException e) { + log.debug(fileSource.getImportSource() + " isn't a valid parser for file"); + } catch (FileMultipleOccurencesException e) { + log.debug("File contains multiple metadata, return with error"); + throw e; + } + } + } + return importRecords; + } + /** * Call destroy on all {@link Destroyable} {@link MetadataSource} objects set in this ImportService */ diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/components/AbstractPlainMetadataSource.java b/dspace-api/src/main/java/org/dspace/importer/external/service/components/AbstractPlainMetadataSource.java new file mode 100644 index 0000000000..45b96f10b9 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/components/AbstractPlainMetadataSource.java @@ -0,0 +1,86 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ + +package org.dspace.importer.external.service.components; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +import org.dspace.importer.external.datamodel.ImportRecord; +import org.dspace.importer.external.exception.FileMultipleOccurencesException; +import org.dspace.importer.external.exception.FileSourceException; +import org.dspace.importer.external.metadatamapping.AbstractMetadataFieldMapping; +import org.dspace.importer.external.metadatamapping.MetadatumDTO; +import org.dspace.importer.external.service.components.dto.PlainMetadataSourceDto; + + +/** + * This class is an abstract implementation of {@link MetadataSource} useful in cases + * of plain metadata sources. + * It provides the methot to mapping metadata to DSpace Format when source is a file + * whit a list of strings. + * + * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it) + */ + +public abstract class AbstractPlainMetadataSource + extends AbstractMetadataFieldMapping + implements FileSource { + + protected abstract List + readData(InputStream fileInpuStream) throws FileSourceException; + + /** + * Return a list of ImportRecord constructed from input file. This list is based on + * the results retrieved from the file (InputStream) parsed through abstract method readData + * + * @param InputStream The inputStream of the file + * @return A list of {@link ImportRecord} + * @throws FileSourceException if, for any reason, the file is not parsable + */ + @Override + public List getRecords(InputStream is) throws FileSourceException { + List datas = readData(is); + List records = new ArrayList<>(); + for (PlainMetadataSourceDto item : datas) { + records.add(toRecord(item)); + } + return records; + } + + /** + * Return an ImportRecord constructed from input file. This list is based on + * the result retrieved from the file (InputStream) parsed through abstract method + * "readData" implementation + * + * @param InputStream The inputStream of the file + * @return An {@link ImportRecord} matching the file content + * @throws FileSourceException if, for any reason, the file is not parsable + * @throws FileMultipleOccurencesException if the file contains more than one entry + */ + @Override + public ImportRecord getRecord(InputStream is) throws FileSourceException, FileMultipleOccurencesException { + List datas = readData(is); + if (datas == null || datas.isEmpty()) { + throw new FileSourceException("File is empty"); + } + if (datas.size() > 1) { + throw new FileMultipleOccurencesException("File " + + "contains more than one entry (" + datas.size() + " entries"); + } + return toRecord(datas.get(0)); + } + + + private ImportRecord toRecord(PlainMetadataSourceDto entry) { + List metadata = new ArrayList<>(); + metadata.addAll(resultToDCValueMapping(entry)); + return new ImportRecord(metadata); + } +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/components/FileSource.java b/dspace-api/src/main/java/org/dspace/importer/external/service/components/FileSource.java new file mode 100644 index 0000000000..ff05b8cdfb --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/components/FileSource.java @@ -0,0 +1,46 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ + +package org.dspace.importer.external.service.components; + +import java.io.InputStream; +import java.util.List; + +import org.dspace.importer.external.datamodel.ImportRecord; +import org.dspace.importer.external.exception.FileMultipleOccurencesException; +import org.dspace.importer.external.exception.FileSourceException; + +/** + * This interface declare the base methods to work with files containing metadata. + * + * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it) + */ +public interface FileSource extends MetadataSource { + + /** + * Return a list of ImportRecord constructed from input file. + * + * @param InputStream The inputStream of the file + * @return A list of {@link ImportRecord} + * @throws FileSourceException if, for any reason, the file is not parsable + */ + public List getRecords(InputStream inputStream) + throws FileSourceException; + + /** + * Return an ImportRecord constructed from input file. + * + * @param InputStream The inputStream of the file + * @return An {@link ImportRecord} matching the file content + * @throws FileSourceException if, for any reason, the file is not parsable + * @throws FileMultipleOccurencesException if the file contains more than one entry + */ + public ImportRecord getRecord(InputStream inputStream) + throws FileSourceException, FileMultipleOccurencesException; + +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/components/MetadataSource.java b/dspace-api/src/main/java/org/dspace/importer/external/service/components/MetadataSource.java index 79bdcfa903..353f77b798 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/service/components/MetadataSource.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/components/MetadataSource.java @@ -8,76 +8,14 @@ package org.dspace.importer.external.service.components; -import java.util.Collection; - -import org.dspace.content.Item; -import org.dspace.importer.external.datamodel.ImportRecord; -import org.dspace.importer.external.datamodel.Query; -import org.dspace.importer.external.exception.MetadataSourceException; - /** - * Common interface for all import implementations. + * Super interface for all import implementations. * * @author Roeland Dillen (roeland at atmire dot com) + * @author Pasquale Cavallo (pasquale.cavallo@4science.it) */ public interface MetadataSource { - /** - * Gets the number of records matching a query - * - * @param query the query in string format - * @return the number of records matching the query - * @throws MetadataSourceException if the underlying methods throw any exception. - */ - public int getNbRecords(String query) throws MetadataSourceException; - /** - * Gets the number of records matching a query - * - * @param query the query object - * @return the number of records matching the query - * @throws MetadataSourceException if the underlying methods throw any exception. - */ - public int getNbRecords(Query query) throws MetadataSourceException; - - /** - * Gets a set of records matching a query. Supports pagination - * - * @param query the query. The query will generally be posted 'as is' to the source - * @param start offset - * @param count page size - * @return a collection of fully transformed id's - * @throws MetadataSourceException if the underlying methods throw any exception. - */ - public Collection getRecords(String query, int start, int count) throws MetadataSourceException; - - /** - * Find records based on a object query. - * - * @param query a query object to base the search on. - * @return a set of records. Fully transformed. - * @throws MetadataSourceException if the underlying methods throw any exception. - */ - public Collection getRecords(Query query) throws MetadataSourceException; - - /** - * Get a single record from the source. - * The first match will be returned - * - * @param id identifier for the record - * @return a matching record - * @throws MetadataSourceException if the underlying methods throw any exception. - */ - public ImportRecord getRecord(String id) throws MetadataSourceException; - - /** - * Get a single record from the source. - * The first match will be returned - * - * @param query a query matching a single record - * @return a matching record - * @throws MetadataSourceException if the underlying methods throw any exception. - */ - public ImportRecord getRecord(Query query) throws MetadataSourceException; /** * The string that identifies this import implementation. Preferable a URI @@ -86,23 +24,4 @@ public interface MetadataSource { */ public String getImportSource(); - /** - * Finds records based on an item - * Delegates to one or more MetadataSource implementations based on the uri. Results will be aggregated. - * - * @param item an item to base the search on - * @return a collection of import records. Only the identifier of the found records may be put in the record. - * @throws MetadataSourceException if the underlying methods throw any exception. - */ - public Collection findMatchingRecords(Item item) throws MetadataSourceException; - - /** - * Finds records based on query object. - * Delegates to one or more MetadataSource implementations based on the uri. Results will be aggregated. - * - * @param query a query object to base the search on. - * @return a collection of import records. Only the identifier of the found records may be put in the record. - * @throws MetadataSourceException passed through. - */ - public Collection findMatchingRecords(Query query) throws MetadataSourceException; } diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/components/QuerySource.java b/dspace-api/src/main/java/org/dspace/importer/external/service/components/QuerySource.java new file mode 100644 index 0000000000..a989365e34 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/components/QuerySource.java @@ -0,0 +1,106 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ + +package org.dspace.importer.external.service.components; + +import java.util.Collection; + +import org.dspace.content.Item; +import org.dspace.importer.external.datamodel.ImportRecord; +import org.dspace.importer.external.datamodel.Query; +import org.dspace.importer.external.exception.MetadataSourceException; + + +/** + * Common interface for database-based imports. + * + * @author Roeland Dillen (roeland at atmire dot com) + * @author Pasquale Cavallo (pasquale.cavallo@4science.it) + */ + +public interface QuerySource extends MetadataSource { + + /** + * Get a single record from the source. + * The first match will be returned + * + * @param id identifier for the record + * @return a matching record + * @throws MetadataSourceException if the underlying methods throw any exception. + */ + public ImportRecord getRecord(String id) throws MetadataSourceException; + + /** + * Gets the number of records matching a query + * + * @param query the query in string format + * @return the number of records matching the query + * @throws MetadataSourceException if the underlying methods throw any exception. + */ + public int getNbRecords(String query) throws MetadataSourceException; + + /** + * Gets the number of records matching a query + * + * @param query the query object + * @return the number of records matching the query + * @throws MetadataSourceException if the underlying methods throw any exception. + */ + public int getNbRecords(Query query) throws MetadataSourceException; + + /** + * Gets a set of records matching a query. Supports pagination + * + * @param query the query. The query will generally be posted 'as is' to the source + * @param start offset + * @param count page size + * @return a collection of fully transformed id's + * @throws MetadataSourceException if the underlying methods throw any exception. + */ + public Collection getRecords(String query, int start, int count) throws MetadataSourceException; + + /** + * Find records based on a object query. + * + * @param query a query object to base the search on. + * @return a set of records. Fully transformed. + * @throws MetadataSourceException if the underlying methods throw any exception. + */ + public Collection getRecords(Query query) throws MetadataSourceException; + + /** + * Get a single record from the source. + * The first match will be returned + * + * @param query a query matching a single record + * @return a matching record + * @throws MetadataSourceException if the underlying methods throw any exception. + */ + public ImportRecord getRecord(Query query) throws MetadataSourceException; + + /** + * Finds records based on query object. + * Delegates to one or more MetadataSource implementations based on the uri. Results will be aggregated. + * + * @param query a query object to base the search on. + * @return a collection of import records. Only the identifier of the found records may be put in the record. + * @throws MetadataSourceException passed through. + */ + public Collection findMatchingRecords(Query query) throws MetadataSourceException; + + /** + * Finds records based on an item + * Delegates to one or more MetadataSource implementations based on the uri. Results will be aggregated. + * + * @param item an item to base the search on + * @return a collection of import records. Only the identifier of the found records may be put in the record. + * @throws MetadataSourceException if the underlying methods throw any exception. + */ + public Collection findMatchingRecords(Item item) throws MetadataSourceException; + +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/components/dto/PlainMetadataKeyValueItem.java b/dspace-api/src/main/java/org/dspace/importer/external/service/components/dto/PlainMetadataKeyValueItem.java new file mode 100644 index 0000000000..c3134db3a5 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/components/dto/PlainMetadataKeyValueItem.java @@ -0,0 +1,37 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.importer.external.service.components.dto; + +/** + * Simple object to construct items + * + * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it) + */ + +public class PlainMetadataKeyValueItem { + + private String key; + private String value; + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/components/dto/PlainMetadataSourceDto.java b/dspace-api/src/main/java/org/dspace/importer/external/service/components/dto/PlainMetadataSourceDto.java new file mode 100644 index 0000000000..bf800f1e49 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/components/dto/PlainMetadataSourceDto.java @@ -0,0 +1,32 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.importer.external.service.components.dto; + +import java.util.List; + + +/** + * Simple object used to construct a list of items. + * This type is used in file plain metadata import as RecordType. + * + * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it) + */ + +public class PlainMetadataSourceDto { + + private List metadata; + + public List getMetadata() { + return metadata; + } + + public void setMetadata(List metadata) { + this.metadata = metadata; + } + +} diff --git a/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml b/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml index bbdf085619..b369b3295a 100644 --- a/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml +++ b/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml @@ -21,7 +21,12 @@ - + + + + + + + + + + + diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java index 91d6f0c652..a85f50414b 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java @@ -8,18 +8,16 @@ package org.dspace.app.rest.repository; import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import java.util.UUID; import javax.servlet.http.HttpServletRequest; -import gr.ekt.bte.core.TransformationEngine; -import gr.ekt.bte.core.TransformationSpec; -import gr.ekt.bte.exceptions.BadTransformationSpec; -import gr.ekt.bte.exceptions.MalformedSourceException; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; import org.dspace.app.rest.Parameter; @@ -27,6 +25,7 @@ import org.dspace.app.rest.SearchRestMethod; import org.dspace.app.rest.converter.WorkspaceItemConverter; import org.dspace.app.rest.exception.DSpaceBadRequestException; import org.dspace.app.rest.exception.RepositoryMethodNotImplementedException; +import org.dspace.app.rest.exception.UnprocessableEntityException; import org.dspace.app.rest.model.ErrorRest; import org.dspace.app.rest.model.WorkspaceItemRest; import org.dspace.app.rest.model.patch.Operation; @@ -44,25 +43,24 @@ import org.dspace.authorize.AuthorizeException; import org.dspace.authorize.service.AuthorizeService; import org.dspace.content.Collection; import org.dspace.content.Item; +import org.dspace.content.ItemServiceImpl; +import org.dspace.content.MetadataValue; import org.dspace.content.WorkspaceItem; import org.dspace.content.service.BitstreamFormatService; import org.dspace.content.service.BitstreamService; import org.dspace.content.service.CollectionService; -import org.dspace.content.service.ItemService; import org.dspace.content.service.WorkspaceItemService; import org.dspace.core.Constants; import org.dspace.core.Context; import org.dspace.eperson.EPerson; import org.dspace.eperson.EPersonServiceImpl; import org.dspace.event.Event; +import org.dspace.importer.external.datamodel.ImportRecord; +import org.dspace.importer.external.exception.FileMultipleOccurencesException; +import org.dspace.importer.external.metadatamapping.MetadatumDTO; +import org.dspace.importer.external.service.ImportService; import org.dspace.services.ConfigurationService; import org.dspace.submit.AbstractProcessingStep; -import org.dspace.submit.lookup.DSpaceWorkspaceItemOutputGenerator; -import org.dspace.submit.lookup.MultipleSubmissionLookupDataLoader; -import org.dspace.submit.lookup.SubmissionItemDataLoader; -import org.dspace.submit.lookup.SubmissionLookupOutputGenerator; -import org.dspace.submit.lookup.SubmissionLookupService; -import org.dspace.submit.util.ItemSubmissionLookupDTO; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; @@ -72,10 +70,12 @@ import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.stereotype.Component; import org.springframework.web.multipart.MultipartFile; + /** * This is the repository responsible to manage WorkspaceItem Rest object * * @author Andrea Bollini (andrea.bollini at 4science.it) + * @author Pasquale Cavallo (pasquale.cavallo at 4science.it) */ @Component(WorkspaceItemRest.CATEGORY + "." + WorkspaceItemRest.NAME) public class WorkspaceItemRestRepository extends DSpaceRestRepository @@ -89,7 +89,7 @@ public class WorkspaceItemRestRepository extends DSpaceRestRepository results = new ArrayList<>(); - try { + try (InputStream fileInputStream = new FileInputStream(file)) { String uuid = request.getParameter("collection"); if (StringUtils.isBlank(uuid)) { uuid = configurationService.getProperty("submission.default.collection"); @@ -370,82 +370,20 @@ public class WorkspaceItemRestRepository extends DSpaceRestRepository tmpResult = new ArrayList(); - - TransformationEngine transformationEngine1 = submissionLookupService.getPhase1TransformationEngine(); - TransformationSpec spec = new TransformationSpec(); - // FIXME this is mostly due to the need to test. The BTE framework has an assert statement that check if the - // number of found record is less than the requested and treat 0 as is, instead, the implementation assume - // 0=unlimited this lead to test failure. - // It is unclear if BTE really respect values other than 0/MAX allowing us to put a protection against heavy - // load - spec.setNumberOfRecords(Integer.MAX_VALUE); - if (transformationEngine1 != null) { - MultipleSubmissionLookupDataLoader dataLoader = - (MultipleSubmissionLookupDataLoader) transformationEngine1.getDataLoader(); - - List fileDataLoaders = submissionLookupService.getFileProviders(); - for (String fileDataLoader : fileDataLoaders) { - dataLoader.setFile(file.getAbsolutePath(), fileDataLoader); - - try { - SubmissionLookupOutputGenerator outputGenerator = - (SubmissionLookupOutputGenerator) transformationEngine1.getOutputGenerator(); - outputGenerator.setDtoList(new ArrayList()); - log.debug("BTE transformation is about to start!"); - transformationEngine1.transform(spec); - log.debug("BTE transformation finished!"); - tmpResult.addAll(outputGenerator.getDtoList()); - if (!tmpResult.isEmpty()) { - //exit with the results founded on the first data provided - break; - } - } catch (BadTransformationSpec e1) { - log.error(e1.getMessage(), e1); - } catch (MalformedSourceException e1) { - log.error(e1.getMessage(), e1); - } - } - } - List result = null; - - //try to ingest workspaceitems - if (!tmpResult.isEmpty()) { - TransformationEngine transformationEngine2 = submissionLookupService.getPhase2TransformationEngine(); - if (transformationEngine2 != null) { - SubmissionItemDataLoader dataLoader = - (SubmissionItemDataLoader) transformationEngine2.getDataLoader(); - dataLoader.setDtoList(tmpResult); - // dataLoader.setProviders() - - DSpaceWorkspaceItemOutputGenerator outputGenerator = - (DSpaceWorkspaceItemOutputGenerator) transformationEngine2.getOutputGenerator(); - outputGenerator.setCollection(collection); - outputGenerator.setContext(context); - outputGenerator.setFormName(submissionConfig.getSubmissionName()); - outputGenerator.setDto(tmpResult.get(0)); - - try { - transformationEngine2.transform(spec); - result = outputGenerator.getWitems(); - } catch (BadTransformationSpec e1) { - e1.printStackTrace(); - } catch (MalformedSourceException e1) { - e1.printStackTrace(); - } - } - } - - //we have to create the workspaceitem to push the file also if nothing found before - if (result == null) { - WorkspaceItem source = - submissionService.createWorkspaceItem(context, getRequestService().getCurrentRequest()); - result = new ArrayList<>(); - result.add(source); + ImportRecord record = null; + try { + record = importService.getRecord(fileInputStream); + } catch (FileMultipleOccurencesException e) { + throw new UnprocessableEntityException("Too many entries in file"); + } catch ( Exception e) { + throw e; } + WorkspaceItem source = submissionService. + createWorkspaceItem(context, getRequestService().getCurrentRequest()); + merge(context, record, source); + result = new ArrayList<>(); + result.add(source); //perform upload of bitstream if there is exact one result and convert workspaceitem to entity rest if (result != null && !result.isEmpty()) { @@ -541,4 +479,22 @@ public class WorkspaceItemRestRepository extends DSpaceRestRepository getPKClass() { return Integer.class; } + + private void merge(Context context, ImportRecord record, WorkspaceItem item) throws SQLException { + for (MetadataValue metadataValue : itemService.getMetadata( + item.getItem(), Item.ANY, Item.ANY, Item.ANY, Item.ANY)) { + itemService.clearMetadata(context, item.getItem(), + metadataValue.getMetadataField().getMetadataSchema().getNamespace(), + metadataValue.getMetadataField().getElement(), + metadataValue.getMetadataField().getQualifier(), + metadataValue.getLanguage()); + } + if (record != null && record.getValueList() != null) { + for (MetadatumDTO metadataValue : record.getValueList()) { + itemService.addMetadata(context, item.getItem(), metadataValue.getSchema(), + metadataValue.getElement(), metadataValue.getQualifier(), null, + metadataValue.getValue()); + } + } + } } diff --git a/dspace-server-webapp/src/test/resources/org/dspace/app/rest/bibtex-test-3-entries.bib b/dspace-server-webapp/src/test/resources/org/dspace/app/rest/bibtex-test-3-entries.bib new file mode 100644 index 0000000000..4d197ff90f --- /dev/null +++ b/dspace-server-webapp/src/test/resources/org/dspace/app/rest/bibtex-test-3-entries.bib @@ -0,0 +1,14 @@ +@misc{ Nobody01, + author = "Nobody Jr", + title = "My Article", + year = "2006" } + +@misc{ Nobody02, + author = "Nobody Jr", + title = "My Article 2", + year = "2006" } + +@misc{ Nobody03, + author = "Nobody Jr", + title = "My Article 3", + year = "2018" } diff --git a/dspace-server-webapp/src/test/resources/org/dspace/app/rest/bibtex-test.bib b/dspace-server-webapp/src/test/resources/org/dspace/app/rest/bibtex-test.bib index 4d197ff90f..d6e0d992a4 100644 --- a/dspace-server-webapp/src/test/resources/org/dspace/app/rest/bibtex-test.bib +++ b/dspace-server-webapp/src/test/resources/org/dspace/app/rest/bibtex-test.bib @@ -1,14 +1,4 @@ @misc{ Nobody01, author = "Nobody Jr", title = "My Article", - year = "2006" } - -@misc{ Nobody02, - author = "Nobody Jr", - title = "My Article 2", - year = "2006" } - -@misc{ Nobody03, - author = "Nobody Jr", - title = "My Article 3", - year = "2018" } + year = "2006" } \ No newline at end of file diff --git a/dspace/config/spring/api/bibtex-integration.xml b/dspace/config/spring/api/bibtex-integration.xml new file mode 100644 index 0000000000..dbf6f6b493 --- /dev/null +++ b/dspace/config/spring/api/bibtex-integration.xml @@ -0,0 +1,74 @@ + + + + + + + Defines which metadatum is mapped on which metadatum. Note that while the key must be unique it + only matters here for postprocessing of the value. The mapped MetadatumContributor has full control over + what metadatafield is generated. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dspace/config/spring/api/external-services.xml b/dspace/config/spring/api/external-services.xml index 520c21a963..08c87d6c74 100644 --- a/dspace/config/spring/api/external-services.xml +++ b/dspace/config/spring/api/external-services.xml @@ -20,10 +20,13 @@ - + - + + + + From aef91df802af0144df8dcc0d040c664d9170741d Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Mon, 22 Jun 2020 15:43:48 +0200 Subject: [PATCH 02/16] Use metadataField as unnamed bean --- .../config/spring/api/bibtex-integration.xml | 53 +++++++++---------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/dspace/config/spring/api/bibtex-integration.xml b/dspace/config/spring/api/bibtex-integration.xml index dbf6f6b493..eade932459 100644 --- a/dspace/config/spring/api/bibtex-integration.xml +++ b/dspace/config/spring/api/bibtex-integration.xml @@ -25,50 +25,47 @@ - + + + + + - + + + + + - + + + + + - + + + + + - + + + + + - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file From 8ad4b0ed39252e9e58adde4f6ac062df2a43c16e Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Tue, 23 Jun 2020 01:51:59 +0200 Subject: [PATCH 03/16] Implement FileSource in PubmedImportMetadataSourceServiceImpl --- ...PubmedImportMetadataSourceServiceImpl.java | 54 +++++-- .../external/service/ImportService.java | 8 +- .../rest/WorkspaceItemRestRepositoryIT.java | 141 +++++++++++++++--- .../config/spring/api/bibtex-integration.xml | 53 +++---- 4 files changed, 195 insertions(+), 61 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/importer/external/pubmed/service/PubmedImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/pubmed/service/PubmedImportMetadataSourceServiceImpl.java index 02d205914a..858953da1e 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/pubmed/service/PubmedImportMetadataSourceServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/pubmed/service/PubmedImportMetadataSourceServiceImpl.java @@ -8,6 +8,10 @@ package org.dspace.importer.external.pubmed.service; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; import java.io.StringReader; import java.util.Collection; import java.util.LinkedList; @@ -20,6 +24,7 @@ import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import com.google.common.io.CharStreams; import org.apache.axiom.om.OMElement; import org.apache.axiom.om.OMXMLBuilderFactory; import org.apache.axiom.om.OMXMLParserWrapper; @@ -27,8 +32,11 @@ import org.apache.axiom.om.xpath.AXIOMXPath; import org.dspace.content.Item; import org.dspace.importer.external.datamodel.ImportRecord; import org.dspace.importer.external.datamodel.Query; +import org.dspace.importer.external.exception.FileMultipleOccurencesException; +import org.dspace.importer.external.exception.FileSourceException; import org.dspace.importer.external.exception.MetadataSourceException; import org.dspace.importer.external.service.AbstractImportMetadataSourceService; +import org.dspace.importer.external.service.components.FileSource; import org.dspace.importer.external.service.components.QuerySource; import org.jaxen.JaxenException; @@ -38,7 +46,7 @@ import org.jaxen.JaxenException; * @author Roeland Dillen (roeland at atmire dot com) */ public class PubmedImportMetadataSourceServiceImpl extends AbstractImportMetadataSourceService - implements QuerySource { + implements QuerySource, FileSource { private String baseAddress; @@ -360,7 +368,6 @@ public class PubmedImportMetadataSourceServiceImpl extends AbstractImportMetadat @Override public Collection call() throws Exception { - List records = new LinkedList(); WebTarget getRecordIdsTarget = pubmedWebTarget .queryParam("term", query.getParameterAsClass("term", String.class)); @@ -385,13 +392,42 @@ public class PubmedImportMetadataSourceServiceImpl extends AbstractImportMetadat invocationBuilder = getRecordsTarget.request(MediaType.TEXT_PLAIN_TYPE); response = invocationBuilder.get(); - List omElements = splitToRecords(response.readEntity(String.class)); - - for (OMElement record : omElements) { - records.add(transformSourceRecords(record)); - } - - return records; + String xml = response.readEntity(String.class); + return parseXMLString(xml); } } + + + @Override + public List getRecords(InputStream inputStream) throws FileSourceException { + String xml = null; + try (Reader reader = new InputStreamReader(inputStream)) { + xml = CharStreams.toString(reader); + return parseXMLString(xml); + } catch (IOException e) { + throw new FileSourceException ("Cannot read XML from InputStream", e); + } + } + + @Override + public ImportRecord getRecord(InputStream inputStream) throws FileSourceException, FileMultipleOccurencesException { + List importRecord = getRecords(inputStream); + if (importRecord == null || importRecord.isEmpty()) { + throw new FileSourceException("Cannot find (valid) record in File"); + } else if (importRecord.size() > 1) { + throw new FileMultipleOccurencesException("File contains more than one entry"); + } else { + return importRecord.get(0); + } + } + + private List parseXMLString(String xml) { + List records = new LinkedList(); + List omElements = splitToRecords(xml); + for (OMElement record : omElements) { + records.add(transformSourceRecords(record)); + } + return records; + } + } diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java b/dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java index fe9f473137..8590fb6114 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java @@ -23,8 +23,8 @@ import org.dspace.importer.external.datamodel.Query; import org.dspace.importer.external.exception.FileMultipleOccurencesException; import org.dspace.importer.external.exception.FileSourceException; import org.dspace.importer.external.exception.MetadataSourceException; -import org.dspace.importer.external.service.components.AbstractPlainMetadataSource; import org.dspace.importer.external.service.components.Destroyable; +import org.dspace.importer.external.service.components.FileSource; import org.dspace.importer.external.service.components.MetadataSource; import org.dspace.importer.external.service.components.QuerySource; import org.springframework.beans.factory.annotation.Autowired; @@ -305,13 +305,13 @@ public class ImportService implements Destroyable { public ImportRecord getRecord(InputStream fileInputStream) throws FileMultipleOccurencesException { ImportRecord importRecords = null; for (MetadataSource metadataSource : importSources.values()) { - if (metadataSource instanceof AbstractPlainMetadataSource) { - AbstractPlainMetadataSource fileSource = (AbstractPlainMetadataSource)metadataSource; + if (metadataSource instanceof FileSource) { + FileSource fileSource = (FileSource)metadataSource; try { importRecords = fileSource.getRecord(fileInputStream); break; } catch (FileSourceException e) { - log.debug(fileSource.getImportSource() + " isn't a valid parser for file"); + log.debug(metadataSource.getImportSource() + " isn't a valid parser for file"); } catch (FileMultipleOccurencesException e) { log.debug("File contains multiple metadata, return with error"); throw e; diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java index 3b95b18ee0..7a78a85a90 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java @@ -871,7 +871,7 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration * * @throws Exception */ - public void createMultipleWorkspaceItemFromFileTest() throws Exception { + public void createSingleWorkspaceItemFromFileTest() throws Exception { context.turnOffAuthorisationSystem(); //** GIVEN ** @@ -907,17 +907,7 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration is("My Article"))) .andExpect( jsonPath("$._embedded.workspaceitems[0]._embedded.collection.id", is(col1.getID().toString()))) - .andExpect(jsonPath("$._embedded.workspaceitems[1].sections.traditionalpageone['dc.title'][0].value", - is("My Article 2"))) - .andExpect( - jsonPath("$._embedded.workspaceitems[1]._embedded.collection.id", is(col1.getID().toString()))) - .andExpect(jsonPath("$._embedded.workspaceitems[2].sections.traditionalpageone['dc.title'][0].value", - is("My Article 3"))) - .andExpect( - jsonPath("$._embedded.workspaceitems[2]._embedded.collection.id", is(col1.getID().toString()))) - .andExpect( - jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()) - ; + jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()); // bulk create workspaceitems explicitly in the col2 getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") @@ -928,21 +918,126 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration is("My Article"))) .andExpect( jsonPath("$._embedded.workspaceitems[0]._embedded.collection.id", is(col2.getID().toString()))) - .andExpect(jsonPath("$._embedded.workspaceitems[1].sections.traditionalpageone['dc.title'][0].value", - is("My Article 2"))) - .andExpect( - jsonPath("$._embedded.workspaceitems[1]._embedded.collection.id", is(col2.getID().toString()))) - .andExpect(jsonPath("$._embedded.workspaceitems[2].sections.traditionalpageone['dc.title'][0].value", - is("My Article 3"))) - .andExpect( - jsonPath("$._embedded.workspaceitems[2]._embedded.collection.id", is(col2.getID().toString()))) - .andExpect( - jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()) - ; + jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()); bibtex.close(); } + @Test + /** + * Test the creation of workspaceitems POSTing to the resource collection endpoint a bibtex file + * contains more than one entry. + * + * @throws Exception + */ + public void createMultipleWorkspaceItemsFromFileTest() throws Exception { + context.turnOffAuthorisationSystem(); + + //** GIVEN ** + //1. A community-collection structure with one parent community with sub-community and two collections. + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Community child1 = CommunityBuilder.createSubCommunity(context, parentCommunity) + .withName("Sub Community") + .build(); + Collection col1 = CollectionBuilder.createCollection(context, child1) + .withName("Collection 1") + .withSubmitterGroup(eperson) + .build(); + Collection col2 = CollectionBuilder.createCollection(context, child1) + .withName("Collection 2") + .withSubmitterGroup(eperson) + .build(); + + InputStream bibtex = getClass().getResourceAsStream("bibtex-test-3-entries.bib"); + final MockMultipartFile bibtexFile = new MockMultipartFile("file", "bibtex-test-3-entries.bib", + "application/x-bibtex", + bibtex); + + context.restoreAuthSystemState(); + + String authToken = getAuthToken(eperson.getEmail(), password); + // bulk create workspaceitems in the default collection (col1) + getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") + .file(bibtexFile)) + // bulk create should return 200, 201 (created) is better for single resource + .andExpect(status().is(422)); + bibtex.close(); + } + + @Test + /** + * Test the creation of workspaceitems POSTing to the resource collection endpoint a pubmed XML + * file. + * + * @throws Exception + */ + public void createPubmedWorkspaceItemFromFileTest() throws Exception { + context.turnOffAuthorisationSystem(); + + //** GIVEN ** + //1. A community-collection structure with one parent community with sub-community and two collections. + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Community child1 = CommunityBuilder.createSubCommunity(context, parentCommunity) + .withName("Sub Community") + .build(); + Collection col1 = CollectionBuilder.createCollection(context, child1) + .withName("Collection 1") + .withSubmitterGroup(eperson) + .build(); + Collection col2 = CollectionBuilder.createCollection(context, child1) + .withName("Collection 2") + .withSubmitterGroup(eperson) + .build(); + InputStream xmlIS = getClass().getResourceAsStream("pubmed-test.xml"); + final MockMultipartFile pubmedFile = new MockMultipartFile("file", "pubmed-test.xml", + "application/xml", xmlIS); + + context.restoreAuthSystemState(); + + String authToken = getAuthToken(eperson.getEmail(), password); + // bulk create workspaceitems in the default collection (col1) + getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") + .file(pubmedFile)) + // bulk create should return 200, 201 (created) is better for single resource + .andExpect(status().isOk()) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.title'][0].value", + is("Multistep microreactions with proteins using electrocapture technology."))) + .andExpect( + jsonPath( + "$._embedded.workspaceitems[0].sections.traditionalpageone['dc.identifier.other'][0].value", + is(15117179))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone" + + "['dc.contributor.author'][0].value", + is("Astorga-Wells, J"))) + .andExpect( + jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.date.issued'][0].value", + is("2004-05-01"))); + + // bulk create workspaceitems explicitly in the col2 + getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") + .file(pubmedFile) + .param("collection", col2.getID().toString())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.title'][0].value", + is("Multistep microreactions with proteins using electrocapture technology."))) + .andExpect( + jsonPath( + "$._embedded.workspaceitems[0].sections.traditionalpageone['dc.identifier.other'][0].value", + is(15117179))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone" + + "['dc.contributor.author'][0].value", + is("Astorga-Wells, J"))) + .andExpect( + jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.date.issued'][0].value", + is("2004-05-01"))); + + xmlIS.close(); + } + @Test /** * Test the creation of a workspaceitem POSTing to the resource collection endpoint a PDF file. As a single item diff --git a/dspace/config/spring/api/bibtex-integration.xml b/dspace/config/spring/api/bibtex-integration.xml index eade932459..9675ef82b3 100644 --- a/dspace/config/spring/api/bibtex-integration.xml +++ b/dspace/config/spring/api/bibtex-integration.xml @@ -25,47 +25,50 @@ - - - - - + - - - - - + - - - - - + - - - - - + - - - - - + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From 3c4a46377175fc7caeeacb55f653880b7f7f4d4f Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Tue, 23 Jun 2020 01:58:42 +0200 Subject: [PATCH 04/16] pubmed test data --- .../org/dspace/app/rest/pubmed-test.xml | 151 ++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 dspace-server-webapp/src/test/resources/org/dspace/app/rest/pubmed-test.xml diff --git a/dspace-server-webapp/src/test/resources/org/dspace/app/rest/pubmed-test.xml b/dspace-server-webapp/src/test/resources/org/dspace/app/rest/pubmed-test.xml new file mode 100644 index 0000000000..3fdceb3880 --- /dev/null +++ b/dspace-server-webapp/src/test/resources/org/dspace/app/rest/pubmed-test.xml @@ -0,0 +1,151 @@ + + + + + + 15117179 + + 2005 + 02 + 15 + + + 2006 + 11 + 15 + +
+ + 0003-2700 + + 76 + 9 + + 2004 + May + 01 + + + Analytical chemistry + Anal. Chem. + + Multistep microreactions with proteins using electrocapture technology. + + 2425-9 + + + A method to perform multistep reactions by means of electroimmobilization of a target molecule in a microflow stream is presented. A target protein is captured by the opposing effects between the hydrodynamic and electric forces, after which another medium is injected into the system. The second medium carries enzymes or other reagents, which are brought into contact with the target protein and react. The immobilization is reversed by disconnecting the electric field, upon which products are collected at the outlet of the device for analysis. On-line reduction, alkylation, and trypsin digestion of proteins is demonstrated and was monitored by MALDI mass spectrometry. + + + + Astorga-Wells + Juan + J + + Department of Medical Biochemistry and Biophysics, Karolinska Institutet, SE-171 77 Stockholm, Sweden. + + + + Bergman + Tomas + T + + + Jörnvall + Hans + H + + + eng + + Journal Article + Research Support, Non-U.S. Gov't + +
+ + United States + Anal Chem + 0370536 + 0003-2700 + + + + 0 + Proteins + + + EC 3.4.21.4 + Trypsin + + + IM + + + Animals + + + Cattle + + + Electrochemistry + + + Horses + + + Microfluidics + instrumentation + methods + + + Peptide Mapping + methods + + + Proteins + analysis + chemistry + + + Spectrometry, Mass, Matrix-Assisted Laser Desorption-Ionization + methods + + + Trypsin + chemistry + + + + + + + 2004 + 5 + 1 + 5 + 0 + + + 2005 + 2 + 16 + 9 + 0 + + + 2004 + 5 + 1 + 5 + 0 + + + ppublish + + 15117179 + 10.1021/ac0354342 + + + + + \ No newline at end of file From 530a9d5fac9eee6fdaa075619087fa519fa11b3d Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Tue, 23 Jun 2020 17:11:56 +0200 Subject: [PATCH 05/16] Tests and minor fix --- ...PubmedImportMetadataSourceServiceImpl.java | 2 +- .../external/service/ImportService.java | 23 +-- .../rest/repository/DSpaceRestRepository.java | 2 +- .../WorkspaceItemRestRepository.java | 146 +++++++++--------- .../rest/WorkspaceItemRestRepositoryIT.java | 31 ++-- .../config/spring/api/external-services.xml | 7 +- 6 files changed, 105 insertions(+), 106 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/importer/external/pubmed/service/PubmedImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/pubmed/service/PubmedImportMetadataSourceServiceImpl.java index 858953da1e..74bc6a8b45 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/pubmed/service/PubmedImportMetadataSourceServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/pubmed/service/PubmedImportMetadataSourceServiceImpl.java @@ -401,7 +401,7 @@ public class PubmedImportMetadataSourceServiceImpl extends AbstractImportMetadat @Override public List getRecords(InputStream inputStream) throws FileSourceException { String xml = null; - try (Reader reader = new InputStreamReader(inputStream)) { + try (Reader reader = new InputStreamReader(inputStream, "UTF-8")) { xml = CharStreams.toString(reader); return parseXMLString(xml); } catch (IOException e) { diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java b/dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java index 8590fb6114..5a838e8027 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java @@ -8,6 +8,9 @@ package org.dspace.importer.external.service; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; import java.io.InputStream; import java.util.Collection; import java.util.Collections; @@ -302,20 +305,22 @@ public class ImportService implements Destroyable { * @return a single record contains the metadatum * @throws FileMultipleOccurencesException if more than one entry is found */ - public ImportRecord getRecord(InputStream fileInputStream) throws FileMultipleOccurencesException { + public ImportRecord getRecord(File file) throws FileMultipleOccurencesException, FileSourceException { ImportRecord importRecords = null; for (MetadataSource metadataSource : importSources.values()) { - if (metadataSource instanceof FileSource) { - FileSource fileSource = (FileSource)metadataSource; - try { + try (InputStream fileInputStream = new FileInputStream(file)) { + if (metadataSource instanceof FileSource) { + FileSource fileSource = (FileSource)metadataSource; importRecords = fileSource.getRecord(fileInputStream); break; - } catch (FileSourceException e) { - log.debug(metadataSource.getImportSource() + " isn't a valid parser for file"); - } catch (FileMultipleOccurencesException e) { - log.debug("File contains multiple metadata, return with error"); - throw e; } + } catch (FileSourceException e) { + log.debug(metadataSource.getImportSource() + " isn't a valid parser for file"); + } catch (FileMultipleOccurencesException e) { + log.debug("File contains multiple metadata, return with error"); + throw e; + } catch (IOException e1) { + throw new FileSourceException("File cannot be read, may be null"); } } return importRecords; diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/DSpaceRestRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/DSpaceRestRepository.java index e8bf235940..5dacf5a61c 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/DSpaceRestRepository.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/DSpaceRestRepository.java @@ -486,7 +486,7 @@ public abstract class DSpaceRestRepository upload(Context context, HttpServletRequest request, - MultipartFile uploadfile) + MultipartFile ... uploadfile) throws SQLException, FileNotFoundException, IOException, AuthorizeException { throw new RepositoryMethodNotImplementedException("No implementation found; Method not allowed!", ""); } diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java index a85f50414b..5827d8987e 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java @@ -8,10 +8,8 @@ package org.dspace.app.rest.repository; import java.io.File; -import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; -import java.io.InputStream; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; @@ -350,85 +348,85 @@ public class WorkspaceItemRestRepository extends DSpaceRestRepository upload(Context context, HttpServletRequest request, - MultipartFile uploadfile) + MultipartFile... uploadfiles) throws SQLException, FileNotFoundException, IOException, AuthorizeException { - File file = Utils.getFile(uploadfile, "upload-loader", "filedataloader"); List results = new ArrayList<>(); - try (InputStream fileInputStream = new FileInputStream(file)) { - String uuid = request.getParameter("collection"); - if (StringUtils.isBlank(uuid)) { - uuid = configurationService.getProperty("submission.default.collection"); + String uuid = request.getParameter("owningCollection"); + if (StringUtils.isBlank(uuid)) { + uuid = configurationService.getProperty("submission.default.collection"); + } + Collection collection = null; + if (StringUtils.isNotBlank(uuid)) { + collection = collectionService.find(context, UUID.fromString(uuid)); + } else { + collection = collectionService.findAuthorizedOptimized(context, Constants.ADD).get(0); + } + + SubmissionConfig submissionConfig = + submissionConfigReader.getSubmissionConfigByCollection(collection.getHandle()); + List result = null; + List records = null; + try { + for (MultipartFile mpFile : uploadfiles) { + File file = Utils.getFile(mpFile, "upload-loader", "filedataloader"); + try { + if (records == null) { + records = new ArrayList<>(); + } + records.add(importService.getRecord(file)); + } finally { + file.delete(); + } } + } catch (FileMultipleOccurencesException e) { + throw new UnprocessableEntityException("Too many entries in file"); + } catch (Exception e) { + log.error("Error importing metadata", e); + } + WorkspaceItem source = submissionService. + createWorkspaceItem(context, getRequestService().getCurrentRequest()); + merge(context, records, source); + result = new ArrayList<>(); + result.add(source); - Collection collection = null; - if (StringUtils.isNotBlank(uuid)) { - collection = collectionService.find(context, UUID.fromString(uuid)); - } else { - collection = collectionService.findAuthorizedOptimized(context, Constants.ADD).get(0); - } - - SubmissionConfig submissionConfig = - submissionConfigReader.getSubmissionConfigByCollection(collection.getHandle()); - List result = null; - ImportRecord record = null; - try { - record = importService.getRecord(fileInputStream); - } catch (FileMultipleOccurencesException e) { - throw new UnprocessableEntityException("Too many entries in file"); - } catch ( Exception e) { - throw e; - } - WorkspaceItem source = submissionService. - createWorkspaceItem(context, getRequestService().getCurrentRequest()); - merge(context, record, source); - result = new ArrayList<>(); - result.add(source); - - //perform upload of bitstream if there is exact one result and convert workspaceitem to entity rest - if (result != null && !result.isEmpty()) { - for (WorkspaceItem wi : result) { - - List errors = new ArrayList(); - - //load bitstream into bundle ORIGINAL only if there is one result (approximately this is the - // right behaviour for pdf file but not for other bibliographic format e.g. bibtex) - if (result.size() == 1) { - - for (int i = 0; i < submissionConfig.getNumberOfSteps(); i++) { - SubmissionStepConfig stepConfig = submissionConfig.getStep(i); - - ClassLoader loader = this.getClass().getClassLoader(); - Class stepClass; - try { - stepClass = loader.loadClass(stepConfig.getProcessingClassName()); - - Object stepInstance = stepClass.newInstance(); - if (UploadableStep.class.isAssignableFrom(stepClass)) { - UploadableStep uploadableStep = (UploadableStep) stepInstance; - ErrorRest err = uploadableStep.upload(context, submissionService, stepConfig, wi, - uploadfile); + //perform upload of bitstream if there is exact one result and convert workspaceitem to entity rest + if (result != null && !result.isEmpty()) { + for (WorkspaceItem wi : result) { + List errors = new ArrayList(); + //load bitstream into bundle ORIGINAL only if there is one result (approximately this is the + // right behaviour for pdf file but not for other bibliographic format e.g. bibtex) + if (result.size() == 1) { + for (int i = 0; i < submissionConfig.getNumberOfSteps(); i++) { + SubmissionStepConfig stepConfig = submissionConfig.getStep(i); + ClassLoader loader = this.getClass().getClassLoader(); + Class stepClass; + try { + stepClass = loader.loadClass(stepConfig.getProcessingClassName()); + Object stepInstance = stepClass.newInstance(); + if (UploadableStep.class.isAssignableFrom(stepClass)) { + UploadableStep uploadableStep = (UploadableStep) stepInstance; + for (MultipartFile mpFile : uploadfiles) { + ErrorRest err = uploadableStep.upload(context, + submissionService, stepConfig, wi, mpFile); if (err != null) { errors.add(err); } } - - } catch (Exception e) { - log.error(e.getMessage(), e); } + } catch (Exception e) { + log.error(e.getMessage(), e); } } - WorkspaceItemRest wsi = converter.toRest(wi, utils.obtainProjection()); - if (result.size() == 1) { - if (!errors.isEmpty()) { - wsi.getErrors().addAll(errors); - } - } - results.add(wsi); } + WorkspaceItemRest wsi = converter.toRest(wi, utils.obtainProjection()); + if (result.size() == 1) { + if (!errors.isEmpty()) { + wsi.getErrors().addAll(errors); + } + } + results.add(wsi); } - } finally { - file.delete(); } return results; } @@ -480,7 +478,7 @@ public class WorkspaceItemRestRepository extends DSpaceRestRepository records, WorkspaceItem item) throws SQLException { for (MetadataValue metadataValue : itemService.getMetadata( item.getItem(), Item.ANY, Item.ANY, Item.ANY, Item.ANY)) { itemService.clearMetadata(context, item.getItem(), @@ -489,11 +487,13 @@ public class WorkspaceItemRestRepository extends DSpaceRestRepository - + - + - - - From ecefb1a30222dbf664f5f3fe59245274111d9875 Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Wed, 24 Jun 2020 14:52:33 +0200 Subject: [PATCH 06/16] Comment and minor improvements --- .../SimpleMetadataContributor.java | 27 +++++++++---------- .../AbstractImportMetadataSourceService.java | 3 ++- .../dto/PlainMetadataKeyValueItem.java | 15 ++++++++++- .../dto/PlainMetadataSourceDto.java | 6 +++++ .../spring-dspace-addon-import-services.xml | 7 ----- .../rest/WorkspaceItemRestRepositoryIT.java | 2 +- 6 files changed, 35 insertions(+), 25 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/SimpleMetadataContributor.java b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/SimpleMetadataContributor.java index 8fcbfdb8b8..21dd1bfcee 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/SimpleMetadataContributor.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/SimpleMetadataContributor.java @@ -19,9 +19,10 @@ import org.dspace.importer.external.service.components.dto.PlainMetadataKeyValue import org.dspace.importer.external.service.components.dto.PlainMetadataSourceDto; /** - * Metadata contributor that takes an PlainMetadataSourceDto instance and turns it into a metadatum + * Metadata contributor that takes an PlainMetadataSourceDto instance and turns it into a + * collection of metadatum * - * @author Roeland Dillen (roeland at atmire dot com) + * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it) */ public class SimpleMetadataContributor implements MetadataContributor { @@ -57,25 +58,21 @@ public class SimpleMetadataContributor implements MetadataContributor contributeMetadata(PlainMetadataSourceDto t) { List values = new LinkedList<>(); - try { - for (PlainMetadataKeyValueItem metadatum : t.getMetadata()) { - if (metadatum.getKey().equals(key)) { - MetadatumDTO dcValue = new MetadatumDTO(); - dcValue.setValue(metadatum.getValue()); - dcValue.setElement(field.getElement()); - dcValue.setQualifier(field.getQualifier()); - dcValue.setSchema(field.getSchema()); - values.add(dcValue); - } + for (PlainMetadataKeyValueItem metadatum : t.getMetadata()) { + if (key.equals(metadatum.getKey())) { + MetadatumDTO dcValue = new MetadatumDTO(); + dcValue.setValue(metadatum.getValue()); + dcValue.setElement(field.getElement()); + dcValue.setQualifier(field.getQualifier()); + dcValue.setSchema(field.getSchema()); + values.add(dcValue); } - } catch (Exception e) { - throw new RuntimeException(e); } return values; } diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/AbstractImportMetadataSourceService.java b/dspace-api/src/main/java/org/dspace/importer/external/service/AbstractImportMetadataSourceService.java index f904ce613c..a803958a9d 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/service/AbstractImportMetadataSourceService.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/AbstractImportMetadataSourceService.java @@ -28,7 +28,8 @@ import org.springframework.beans.factory.annotation.Required; * * @author Roeland Dillen (roeland at atmire dot com) */ -public abstract class AbstractImportMetadataSourceService extends AbstractRemoteMetadataSource { +public abstract class AbstractImportMetadataSourceService extends AbstractRemoteMetadataSource + implements MetadataSource { private GenerateQueryService generateQueryForItem = null; private MetadataFieldMapping> metadataFieldMapping; diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/components/dto/PlainMetadataKeyValueItem.java b/dspace-api/src/main/java/org/dspace/importer/external/service/components/dto/PlainMetadataKeyValueItem.java index c3134db3a5..fa362760b9 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/service/components/dto/PlainMetadataKeyValueItem.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/components/dto/PlainMetadataKeyValueItem.java @@ -12,24 +12,37 @@ package org.dspace.importer.external.service.components.dto; * * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it) */ - public class PlainMetadataKeyValueItem { private String key; private String value; + /* + * In a key-value items, like PlainMetadata, this method get the item's key + */ public String getKey() { return key; } + /* + * In a key-value items, like PlainMetadata, this method set the item's key. + * Never set or leave this field to null + * + */ public void setKey(String key) { this.key = key; } + /* + * In key-value items, like PlainMetadata, this method get the item's value + */ public String getValue() { return value; } + /* + * In key-value items, like PlainMetadata, this method set the item's value + */ public void setValue(String value) { this.value = value; } diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/components/dto/PlainMetadataSourceDto.java b/dspace-api/src/main/java/org/dspace/importer/external/service/components/dto/PlainMetadataSourceDto.java index bf800f1e49..041823b027 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/service/components/dto/PlainMetadataSourceDto.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/components/dto/PlainMetadataSourceDto.java @@ -21,10 +21,16 @@ public class PlainMetadataSourceDto { private List metadata; + /* + * Method used to get the Metadata list + */ public List getMetadata() { return metadata; } + /* + * Method used to set the metadata list + */ public void setMetadata(List metadata) { this.metadata = metadata; } diff --git a/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml b/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml index b369b3295a..3b4f7bad31 100644 --- a/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml +++ b/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml @@ -53,13 +53,6 @@ - - - diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java index 297282980c..ba5f3226d0 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java @@ -900,7 +900,7 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration String authToken = getAuthToken(eperson.getEmail(), password); // bulk create workspaceitems in the default collection (col1) getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") - .file(bibtexFile).param("owningCollection", col1.getID().toString())) + .file(bibtexFile).param("projection", "full")) // bulk create should return 200, 201 (created) is better for single resource .andExpect(status().isOk()) .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.title'][0].value", From 9208879abffaf2866a3e7460ad3a4f32c0fc5f00 Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Thu, 25 Jun 2020 11:57:05 +0200 Subject: [PATCH 07/16] Update test --- .../rest/WorkspaceItemRestRepositoryIT.java | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java index ba5f3226d0..9dbdbb3138 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java @@ -892,23 +892,29 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration .build(); InputStream bibtex = getClass().getResourceAsStream("bibtex-test.bib"); - final MockMultipartFile bibtexFile = new MockMultipartFile("file", "bibtex-test.bib", "application/x-bibtex", - bibtex); + final MockMultipartFile bibtexFile = new MockMultipartFile("file", "/local/path/bibtex-test.bib", + "application/x-bibtex", bibtex); context.restoreAuthSystemState(); String authToken = getAuthToken(eperson.getEmail(), password); // bulk create workspaceitems in the default collection (col1) getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") - .file(bibtexFile).param("projection", "full")) + .file(bibtexFile)) // bulk create should return 200, 201 (created) is better for single resource .andExpect(status().isOk()) .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.title'][0].value", is("My Article"))) .andExpect( jsonPath("$._embedded.workspaceitems[0]._embedded.collection.id", is(col1.getID().toString()))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.upload.files[0]" + + ".metadata['dc.source'][0].value", + is("/local/path/bibtex-test.bib"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.upload.files[0]" + + ".metadata['dc.title'][0].value", + is("bibtex-test.bib"))) .andExpect( - jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()) + jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()); ; // bulk create workspaceitems explicitly in the col2 @@ -920,6 +926,12 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration is("My Article"))) .andExpect( jsonPath("$._embedded.workspaceitems[0]._embedded.collection.id", is(col2.getID().toString()))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.upload.files[0]" + + ".metadata['dc.source'][0].value", + is("/local/path/bibtex-test.bib"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.upload" + + ".files[0].metadata['dc.title'][0].value", + is("bibtex-test.bib"))) .andExpect( jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()); @@ -996,7 +1008,7 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration .withSubmitterGroup(eperson) .build(); InputStream xmlIS = getClass().getResourceAsStream("pubmed-test.xml"); - final MockMultipartFile pubmedFile = new MockMultipartFile("file", "pubmed-test.xml", + final MockMultipartFile pubmedFile = new MockMultipartFile("file", "/local/path/pubmed-test.xml", "application/xml", xmlIS); context.restoreAuthSystemState(); @@ -1015,7 +1027,13 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration is("15117179"))) .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone" + "['dc.contributor.author'][0].value", - is("Astorga-Wells, Juan"))); + is("Astorga-Wells, Juan"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.upload.files[0]" + + ".metadata['dc.source'][0].value", + is("/local/path/pubmed-test.xml"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.upload.files[0]" + + ".metadata['dc.title'][0].value", + is("pubmed-test.xml"))); // bulk create workspaceitems explicitly in the col2 getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") @@ -1030,7 +1048,11 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration is("15117179"))) .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone" + "['dc.contributor.author'][0].value", - is("Astorga-Wells, Juan"))); + is("Astorga-Wells, Juan"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.upload.files[0].metadata['dc.source'][0].value", + is("/local/path/pubmed-test.xml"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.upload.files[0].metadata['dc.title'][0].value", + is("pubmed-test.xml"))); xmlIS.close(); } From c0b92196f3e1489bac5a475369890c0df3fd60cc Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Thu, 25 Jun 2020 13:43:26 +0200 Subject: [PATCH 08/16] Fix multifile import --- .../app/rest/RestResourceController.java | 2 +- .../rest/repository/DSpaceRestRepository.java | 4 +- .../WorkspaceItemRestRepository.java | 4 +- .../rest/WorkspaceItemRestRepositoryIT.java | 89 +++++++++++++++++++ 4 files changed, 95 insertions(+), 4 deletions(-) diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/RestResourceController.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/RestResourceController.java index a1684d782e..dbd9421bb2 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/RestResourceController.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/RestResourceController.java @@ -616,7 +616,7 @@ public class RestResourceController implements InitializingBean { HttpServletRequest request, @PathVariable String apiCategory, @PathVariable String model, - @RequestParam("file") MultipartFile uploadfile) + @RequestParam("file") List uploadfile) throws SQLException, FileNotFoundException, IOException, AuthorizeException { checkModelPluralForm(apiCategory, model); diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/DSpaceRestRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/DSpaceRestRepository.java index 5dacf5a61c..149855c488 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/DSpaceRestRepository.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/DSpaceRestRepository.java @@ -463,7 +463,7 @@ public abstract class DSpaceRestRepository upload(HttpServletRequest request, MultipartFile uploadfile) + public Iterable upload(HttpServletRequest request, List uploadfile) throws SQLException, FileNotFoundException, IOException, AuthorizeException { Context context = obtainContext(); Iterable entity = upload(context, request, uploadfile); @@ -486,7 +486,7 @@ public abstract class DSpaceRestRepository upload(Context context, HttpServletRequest request, - MultipartFile ... uploadfile) + List uploadfile) throws SQLException, FileNotFoundException, IOException, AuthorizeException { throw new RepositoryMethodNotImplementedException("No implementation found; Method not allowed!", ""); } diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java index 5827d8987e..4798d138b9 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java @@ -348,7 +348,7 @@ public class WorkspaceItemRestRepository extends DSpaceRestRepository upload(Context context, HttpServletRequest request, - MultipartFile... uploadfiles) + List uploadfiles) throws SQLException, FileNotFoundException, IOException, AuthorizeException { List results = new ArrayList<>(); @@ -375,6 +375,7 @@ public class WorkspaceItemRestRepository extends DSpaceRestRepository(); } records.add(importService.getRecord(file)); + break; } finally { file.delete(); } @@ -394,6 +395,7 @@ public class WorkspaceItemRestRepository extends DSpaceRestRepository errors = new ArrayList(); + wi.setMultipleFiles(uploadfiles.size() > 1); //load bitstream into bundle ORIGINAL only if there is one result (approximately this is the // right behaviour for pdf file but not for other bibliographic format e.g. bibtex) if (result.size() == 1) { diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java index 9dbdbb3138..9b097db62c 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java @@ -938,6 +938,95 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration bibtex.close(); } + + + @Test + /** + * Test the creation of workspaceitems POSTing to the resource collection endpoint a + * bibtex and pubmed files + * + * @throws Exception + */ + public void createSingleWorkspaceItemFromMultipleFilesWithOneEntryTest() throws Exception { + context.turnOffAuthorisationSystem(); + //** GIVEN ** + //1. A community-collection structure with one parent community with sub-community and two collections. + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Community child1 = CommunityBuilder.createSubCommunity(context, parentCommunity) + .withName("Sub Community") + .build(); + Collection col1 = CollectionBuilder.createCollection(context, child1) + .withName("Collection 1") + .withSubmitterGroup(eperson) + .build(); + Collection col2 = CollectionBuilder.createCollection(context, child1) + .withName("Collection 2") + .withSubmitterGroup(eperson) + .build(); + + InputStream bibtex = getClass().getResourceAsStream("bibtex-test.bib"); + final MockMultipartFile bibtexFile = new MockMultipartFile("file", "/local/path/bibtex-test.bib", + "application/x-bibtex", bibtex); + InputStream xmlIS = getClass().getResourceAsStream("pubmed-test.xml"); + final MockMultipartFile pubmedFile = new MockMultipartFile("file", "/local/path/pubmed-test.xml", + "application/xml", xmlIS); + + context.restoreAuthSystemState(); + + String authToken = getAuthToken(eperson.getEmail(), password); + // bulk create workspaceitems in the default collection (col1) + getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") + .file(bibtexFile).file(pubmedFile)) + // bulk create should return 200, 201 (created) is better for single resource + .andExpect(status().isOk()) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.title'][0].value", + is("My Article"))) + .andExpect( + jsonPath("$._embedded.workspaceitems[0]._embedded.collection.id", is(col1.getID().toString()))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.upload.files[0]" + + ".metadata['dc.source'][0].value", + is("/local/path/bibtex-test.bib"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.upload.files[0]" + + ".metadata['dc.title'][0].value", + is("bibtex-test.bib"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.title'][1].value") + .doesNotExist()) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.upload.files[1]" + + ".metadata['dc.source'][0].value", + is("/local/path/pubmed-test.xml"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.upload.files[1]" + + ".metadata['dc.title'][0].value", + is("pubmed-test.xml"))); + + // bulk create workspaceitems explicitly in the col2 + getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") + .file(bibtexFile).file(pubmedFile) + .param("owningCollection", col2.getID().toString())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.title'][0].value", + is("My Article"))) + .andExpect( + jsonPath("$._embedded.workspaceitems[0]._embedded.collection.id", is(col2.getID().toString()))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.upload.files[0]" + + ".metadata['dc.source'][0].value", + is("/local/path/bibtex-test.bib"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.upload" + + ".files[0].metadata['dc.title'][0].value", + is("bibtex-test.bib"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.title'][1].value") + .doesNotExist()) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.upload.files[1]" + + ".metadata['dc.source'][0].value", + is("/local/path/pubmed-test.xml"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.upload.files[1]" + + ".metadata['dc.title'][0].value", + is("pubmed-test.xml"))); + bibtex.close(); + xmlIS.close(); + } + @Test /** * Test the creation of workspaceitems POSTing to the resource collection endpoint a bibtex file From 014c623256364e8088d710c48c7c73a7de624db4 Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Thu, 25 Jun 2020 14:15:20 +0200 Subject: [PATCH 09/16] Manage empty or invalid files in multiple submission --- .../app/rest/repository/WorkspaceItemRestRepository.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java index 4798d138b9..e66f67fd9a 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java @@ -374,8 +374,11 @@ public class WorkspaceItemRestRepository extends DSpaceRestRepository(); } - records.add(importService.getRecord(file)); - break; + ImportRecord record = importService.getRecord(file); + if (record != null) { + records.add(record); + break; + } } finally { file.delete(); } From cfc53c98d21b1726c7a921d3ea8f30e1176b2874 Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Wed, 1 Jul 2020 10:52:49 +0200 Subject: [PATCH 10/16] Live import minor changes + LGTM alert fix --- .../rest/repository/WorkspaceItemRestRepository.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java index e66f67fd9a..09a9f8febf 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java @@ -41,12 +41,12 @@ import org.dspace.authorize.AuthorizeException; import org.dspace.authorize.service.AuthorizeService; import org.dspace.content.Collection; import org.dspace.content.Item; -import org.dspace.content.ItemServiceImpl; import org.dspace.content.MetadataValue; import org.dspace.content.WorkspaceItem; import org.dspace.content.service.BitstreamFormatService; import org.dspace.content.service.BitstreamService; import org.dspace.content.service.CollectionService; +import org.dspace.content.service.ItemService; import org.dspace.content.service.WorkspaceItemService; import org.dspace.core.Constants; import org.dspace.core.Context; @@ -87,7 +87,7 @@ public class WorkspaceItemRestRepository extends DSpaceRestRepository result = null; - List records = null; + List records = new ArrayList<>(); try { for (MultipartFile mpFile : uploadfiles) { File file = Utils.getFile(mpFile, "upload-loader", "filedataloader"); try { - if (records == null) { - records = new ArrayList<>(); - } ImportRecord record = importService.getRecord(file); if (record != null) { records.add(record); @@ -395,7 +392,7 @@ public class WorkspaceItemRestRepository extends DSpaceRestRepository errors = new ArrayList(); wi.setMultipleFiles(uploadfiles.size() > 1); From 5cdc55163a00708c3c45924cf699c44fd188660d Mon Sep 17 00:00:00 2001 From: Mykhaylo Date: Mon, 20 Jul 2020 13:06:46 +0200 Subject: [PATCH 11/16] Implement community feedbacks --- ...PubmedImportMetadataSourceServiceImpl.java | 4 ++-- .../external/service/ImportService.java | 9 ++++----- .../service/components/QuerySource.java | 4 ++-- .../rest/WorkspaceItemRestRepositoryIT.java | 19 ++++++++++--------- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/importer/external/pubmed/service/PubmedImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/pubmed/service/PubmedImportMetadataSourceServiceImpl.java index 74bc6a8b45..02f169a2b4 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/pubmed/service/PubmedImportMetadataSourceServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/pubmed/service/PubmedImportMetadataSourceServiceImpl.java @@ -60,7 +60,7 @@ public class PubmedImportMetadataSourceServiceImpl extends AbstractImportMetadat * @throws MetadataSourceException if the underlying methods throw any exception. */ @Override - public int getNbRecords(String query) throws MetadataSourceException { + public int getRecordsCount(String query) throws MetadataSourceException { return retry(new GetNbRecords(query)); } @@ -72,7 +72,7 @@ public class PubmedImportMetadataSourceServiceImpl extends AbstractImportMetadat * @throws MetadataSourceException if the underlying methods throw any exception. */ @Override - public int getNbRecords(Query query) throws MetadataSourceException { + public int getRecordsCount(Query query) throws MetadataSourceException { return retry(new GetNbRecords(query)); } diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java b/dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java index 5a838e8027..643e866576 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java @@ -159,7 +159,7 @@ public class ImportService implements Destroyable { int total = 0; for (MetadataSource metadataSource : matchingImports(uri)) { if (metadataSource instanceof QuerySource) { - total += ((QuerySource)metadataSource).getNbRecords(query); + total += ((QuerySource)metadataSource).getRecordsCount(query); } } return total; @@ -181,7 +181,7 @@ public class ImportService implements Destroyable { int total = 0; for (MetadataSource metadataSource : matchingImports(uri)) { if (metadataSource instanceof QuerySource) { - total += ((QuerySource)metadataSource).getNbRecords(query); + total += ((QuerySource)metadataSource).getRecordsCount(query); } } return total; @@ -297,11 +297,10 @@ public class ImportService implements Destroyable { } /* - * Get a collection of record from InputStream, + * Get a collection of record from File, * The first match will be return. - * This method doesn't close the InputStream. * - * @param fileInputStream the input stream to the resource + * @param file The file from which will read records * @return a single record contains the metadatum * @throws FileMultipleOccurencesException if more than one entry is found */ diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/components/QuerySource.java b/dspace-api/src/main/java/org/dspace/importer/external/service/components/QuerySource.java index a989365e34..bcd10cc554 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/service/components/QuerySource.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/components/QuerySource.java @@ -42,7 +42,7 @@ public interface QuerySource extends MetadataSource { * @return the number of records matching the query * @throws MetadataSourceException if the underlying methods throw any exception. */ - public int getNbRecords(String query) throws MetadataSourceException; + public int getRecordsCount(String query) throws MetadataSourceException; /** * Gets the number of records matching a query @@ -51,7 +51,7 @@ public interface QuerySource extends MetadataSource { * @return the number of records matching the query * @throws MetadataSourceException if the underlying methods throw any exception. */ - public int getNbRecords(Query query) throws MetadataSourceException; + public int getRecordsCount(Query query) throws MetadataSourceException; /** * Gets a set of records matching a query. Supports pagination diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java index c312faa831..294a704234 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java @@ -898,10 +898,10 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration context.restoreAuthSystemState(); String authToken = getAuthToken(eperson.getEmail(), password); - // bulk create workspaceitems in the default collection (col1) + // create workspaceitem in the default collection (col1) getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(bibtexFile)) - // bulk create should return 200, 201 (created) is better for single resource + // create should return 200, 201 (created) is better for single resource .andExpect(status().isOk()) .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.title'][0].value", is("My Article"))) @@ -917,7 +917,7 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()); ; - // bulk create workspaceitems explicitly in the col2 + // create workspaceitem explicitly in the col2 getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(bibtexFile) .param("owningCollection", col2.getID().toString())) @@ -979,7 +979,7 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration // bulk create workspaceitems in the default collection (col1) getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(bibtexFile).file(pubmedFile)) - // bulk create should return 200, 201 (created) is better for single resource + // create should return 200, 201 (created) is better for single resource .andExpect(status().isOk()) .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.title'][0].value", is("My Article"))) @@ -1000,7 +1000,7 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration + ".metadata['dc.title'][0].value", is("pubmed-test.xml"))); - // bulk create workspaceitems explicitly in the col2 + // create workspaceitem explicitly in the col2 getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(bibtexFile).file(pubmedFile) .param("owningCollection", col2.getID().toString())) @@ -1062,10 +1062,11 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration context.restoreAuthSystemState(); String authToken = getAuthToken(eperson.getEmail(), password); - // bulk create workspaceitems in the default collection (col1) + // create workspaceitem in the default collection (col1) getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(bibtexFile)) - // bulk create should return 200, 201 (created) is better for single resource + // create should return return a 422 because we don't allow/support bibliographic files + // that have multiple metadata records .andExpect(status().is(422)); bibtex.close(); } @@ -1103,7 +1104,7 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration context.restoreAuthSystemState(); String authToken = getAuthToken(eperson.getEmail(), password); - // bulk create workspaceitems in the default collection (col1) + // create workspaceitem in the default collection (col1) getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(pubmedFile)) // bulk create should return 200, 201 (created) is better for single resource @@ -1124,7 +1125,7 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration + ".metadata['dc.title'][0].value", is("pubmed-test.xml"))); - // bulk create workspaceitems explicitly in the col2 + // create workspaceitem explicitly in the col2 getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(pubmedFile) .param("owningCollection", col2.getID().toString())) From 575ce02077ac7438e70e8508a25129ac365b2341 Mon Sep 17 00:00:00 2001 From: Mykhaylo Date: Mon, 20 Jul 2020 15:32:32 +0200 Subject: [PATCH 12/16] update comments --- .../app/rest/WorkspaceItemRestRepositoryIT.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java index 294a704234..c51193dacf 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java @@ -898,7 +898,7 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration context.restoreAuthSystemState(); String authToken = getAuthToken(eperson.getEmail(), password); - // create workspaceitem in the default collection (col1) + // create a workspaceitem from a single bibliographic entry file explicitly in the default collection (col1) getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(bibtexFile)) // create should return 200, 201 (created) is better for single resource @@ -917,7 +917,7 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()); ; - // create workspaceitem explicitly in the col2 + // create a workspaceitem from a single bibliographic entry file explicitly in the col2 getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(bibtexFile) .param("owningCollection", col2.getID().toString())) @@ -976,7 +976,7 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration context.restoreAuthSystemState(); String authToken = getAuthToken(eperson.getEmail(), password); - // bulk create workspaceitems in the default collection (col1) + // create a workspaceitem from a single bibliographic entry file explicitly in the default collection (col1) getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(bibtexFile).file(pubmedFile)) // create should return 200, 201 (created) is better for single resource @@ -1000,7 +1000,7 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration + ".metadata['dc.title'][0].value", is("pubmed-test.xml"))); - // create workspaceitem explicitly in the col2 + // create a workspaceitem from a single bibliographic entry file explicitly in the col2 getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(bibtexFile).file(pubmedFile) .param("owningCollection", col2.getID().toString())) @@ -1062,7 +1062,7 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration context.restoreAuthSystemState(); String authToken = getAuthToken(eperson.getEmail(), password); - // create workspaceitem in the default collection (col1) + // create a workspaceitem from a single bibliographic entry file explicitly in the default collection (col1) getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(bibtexFile)) // create should return return a 422 because we don't allow/support bibliographic files @@ -1104,10 +1104,9 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration context.restoreAuthSystemState(); String authToken = getAuthToken(eperson.getEmail(), password); - // create workspaceitem in the default collection (col1) + // create a workspaceitem from a single bibliographic entry file explicitly in the default collection (col1) getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(pubmedFile)) - // bulk create should return 200, 201 (created) is better for single resource .andExpect(status().isOk()) .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.title'][0].value", is("Multistep microreactions with proteins using electrocapture technology."))) @@ -1125,7 +1124,7 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration + ".metadata['dc.title'][0].value", is("pubmed-test.xml"))); - // create workspaceitem explicitly in the col2 + // create a workspaceitem from a single bibliographic entry file explicitly in the col2 getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(pubmedFile) .param("owningCollection", col2.getID().toString())) From 239bc5271d248923cf65df277ccfc9a4a4244860 Mon Sep 17 00:00:00 2001 From: KevinVdV Date: Wed, 22 Jul 2020 11:31:51 +0200 Subject: [PATCH 13/16] [Issue: 2800] Put dynamic workflowgroup links on Collections in an array --- .../CollectionResourceWorkflowGroupHalLinkFactory.java | 4 ++-- .../org/dspace/app/rest/model/hateoas/HALResource.java | 8 -------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/link/CollectionResourceWorkflowGroupHalLinkFactory.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/link/CollectionResourceWorkflowGroupHalLinkFactory.java index 7d0256cfc4..c049a74c0d 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/link/CollectionResourceWorkflowGroupHalLinkFactory.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/link/CollectionResourceWorkflowGroupHalLinkFactory.java @@ -47,9 +47,9 @@ public class CollectionResourceWorkflowGroupHalLinkFactory Map roles = WorkflowUtils.getCollectionRoles(collection); UUID resourceUuid = UUID.fromString(halResource.getContent().getUuid()); for (Map.Entry entry : roles.entrySet()) { - list.add(buildLink("workflowGroups/" + entry.getKey(), getMethodOn() + list.add(buildLink("workflowGroups", getMethodOn() .getWorkflowGroupForRole(resourceUuid, null, null, - entry.getKey()))); + entry.getKey())).withName(entry.getKey())); } } diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/hateoas/HALResource.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/hateoas/HALResource.java index 31e2c672e3..0077ae1bc5 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/hateoas/HALResource.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/hateoas/HALResource.java @@ -44,12 +44,4 @@ public abstract class HALResource extends EntityModel { public void setPageHeader(EmbeddedPageHeader page) { this.pageHeader = page; } - - @Override - public EntityModel add(Link link) { - if (!hasLink(link.getRel())) { - return super.add(link); - } - return this; - } } From 107c70a6eed7e37c74da40c8c1981921d39918d6 Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Wed, 29 Jul 2020 12:22:22 +0200 Subject: [PATCH 14/16] Add extensions supported by FileSource in Live Import --- ...PubmedImportMetadataSourceServiceImpl.java | 17 ++++++++++++- .../external/service/ImportService.java | 12 +++++++--- .../AbstractPlainMetadataSource.java | 17 +++++++++++++ .../service/components/FileSource.java | 24 +++++++++++++++++++ .../spring-dspace-addon-import-services.xml | 12 +++++++++- .../WorkspaceItemRestRepository.java | 2 +- 6 files changed, 78 insertions(+), 6 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/importer/external/pubmed/service/PubmedImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/pubmed/service/PubmedImportMetadataSourceServiceImpl.java index 02f169a2b4..b4d8ee1222 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/pubmed/service/PubmedImportMetadataSourceServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/pubmed/service/PubmedImportMetadataSourceServiceImpl.java @@ -52,6 +52,22 @@ public class PubmedImportMetadataSourceServiceImpl extends AbstractImportMetadat private WebTarget pubmedWebTarget; + private List supportedExtensions; + + /** + * Set the file extensions supported by this metadata service + * + * @param supportedExtensionsthe file extensions (xml,txt,...) supported by this service + */ + public void setSupportedExtensions(List supportedExtensions) { + this.supportedExtensions = supportedExtensions; + } + + @Override + public List getSupportedExtensions() { + return supportedExtensions; + } + /** * Find the number of records matching a query; * @@ -429,5 +445,4 @@ public class PubmedImportMetadataSourceServiceImpl extends AbstractImportMetadat } return records; } - } diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java b/dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java index 643e866576..815a10b5a7 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java @@ -301,18 +301,24 @@ public class ImportService implements Destroyable { * The first match will be return. * * @param file The file from which will read records + * @param originalName The original file name or full path * @return a single record contains the metadatum * @throws FileMultipleOccurencesException if more than one entry is found */ - public ImportRecord getRecord(File file) throws FileMultipleOccurencesException, FileSourceException { + public ImportRecord getRecord(File file, String originalName) + throws FileMultipleOccurencesException, FileSourceException { ImportRecord importRecords = null; for (MetadataSource metadataSource : importSources.values()) { try (InputStream fileInputStream = new FileInputStream(file)) { if (metadataSource instanceof FileSource) { FileSource fileSource = (FileSource)metadataSource; - importRecords = fileSource.getRecord(fileInputStream); - break; + if (fileSource.isValidSourceForFile(originalName)) { + importRecords = fileSource.getRecord(fileInputStream); + break; + } } + //catch statements is required because we could have supported format (i.e. XML) + //which fail on schema validation } catch (FileSourceException e) { log.debug(metadataSource.getImportSource() + " isn't a valid parser for file"); } catch (FileMultipleOccurencesException e) { diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/components/AbstractPlainMetadataSource.java b/dspace-api/src/main/java/org/dspace/importer/external/service/components/AbstractPlainMetadataSource.java index 45b96f10b9..019cf33177 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/service/components/AbstractPlainMetadataSource.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/components/AbstractPlainMetadataSource.java @@ -36,6 +36,23 @@ public abstract class AbstractPlainMetadataSource protected abstract List readData(InputStream fileInpuStream) throws FileSourceException; + + private List supportedExtensions; + + /** + * Set the file extensions supported by this metadata service + * + * @param supportedExtensionsthe file extensions (xml,txt,...) supported by this service + */ + public void setSupportedExtensions(List supportedExtensions) { + this.supportedExtensions = supportedExtensions; + } + + @Override + public List getSupportedExtensions() { + return supportedExtensions; + } + /** * Return a list of ImportRecord constructed from input file. This list is based on * the results retrieved from the file (InputStream) parsed through abstract method readData diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/components/FileSource.java b/dspace-api/src/main/java/org/dspace/importer/external/service/components/FileSource.java index ff05b8cdfb..febe01ee53 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/service/components/FileSource.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/components/FileSource.java @@ -43,4 +43,28 @@ public interface FileSource extends MetadataSource { public ImportRecord getRecord(InputStream inputStream) throws FileSourceException, FileMultipleOccurencesException; + /** + * This method is used to decide if the FileSource manage the file format + * + * @param originalName the file file original name + * @return true if the FileSource can parse the file, false otherwise + */ + public default boolean isValidSourceForFile(String originalName) { + List extensions = getSupportedExtensions(); + if (extensions == null || extensions.isEmpty()) { + return false; + } + if (originalName != null && originalName.contains(".")) { + String extension = originalName.substring(originalName.lastIndexOf('.') + 1, + originalName.length()); + return getSupportedExtensions().contains(extension); + } + return false; + } + + /** + * Get the file extensions (xml, csv, txt, ...) supported by the FileSource implementation + */ + public List getSupportedExtensions(); + } diff --git a/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml b/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml index 3b4f7bad31..c65b004f7e 100644 --- a/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml +++ b/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml @@ -41,6 +41,11 @@ class="org.dspace.importer.external.pubmed.service.PubmedImportMetadataSourceServiceImpl" scope="singleton"> + + + xml + + @@ -50,7 +55,12 @@ - + + + bib + bibtex + + diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java index 3befc00a42..1d6471c6be 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java @@ -380,7 +380,7 @@ public class WorkspaceItemRestRepository extends DSpaceRestRepository Date: Wed, 29 Jul 2020 13:00:16 +0200 Subject: [PATCH 15/16] [Task 72222] fixed a bug with the links after the implementation of the array dynamic workflowgroup links --- .../app/rest/model/hateoas/HALResource.java | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/hateoas/HALResource.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/hateoas/HALResource.java index 0077ae1bc5..d82da2b183 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/hateoas/HALResource.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/hateoas/HALResource.java @@ -8,11 +8,13 @@ package org.dspace.app.rest.model.hateoas; import java.util.HashMap; +import java.util.List; import java.util.Map; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonUnwrapped; +import org.apache.commons.lang3.StringUtils; import org.springframework.hateoas.EntityModel; import org.springframework.hateoas.Link; @@ -44,4 +46,26 @@ public abstract class HALResource extends EntityModel { public void setPageHeader(EmbeddedPageHeader page) { this.pageHeader = page; } + + @Override + public EntityModel add(Link link) { + if (!hasLink(link.getRel())) { + return super.add(link); + } else { + String name = link.getName(); + if (StringUtils.isNotBlank(name)) { + List list = this.getLinks(link.getRel()); + boolean doesNameExist = false; + for (Link existingLink : list) { + if (StringUtils.equalsIgnoreCase(existingLink.getName(), link.getName())) { + doesNameExist = true; + } + } + if (!doesNameExist) { + super.add(link); + } + } + } + return this; + } } From 61d44087fd6d8b8e9f035e5962fedcde8ef8ea7b Mon Sep 17 00:00:00 2001 From: Raf Ponsaerts Date: Thu, 6 Aug 2020 12:25:11 +0200 Subject: [PATCH 16/16] [Task 72222] simplified the for loop in halResource to use java8 stream --- .../org/dspace/app/rest/model/hateoas/HALResource.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/hateoas/HALResource.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/hateoas/HALResource.java index d82da2b183..2631b63417 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/hateoas/HALResource.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/hateoas/HALResource.java @@ -55,13 +55,8 @@ public abstract class HALResource extends EntityModel { String name = link.getName(); if (StringUtils.isNotBlank(name)) { List list = this.getLinks(link.getRel()); - boolean doesNameExist = false; - for (Link existingLink : list) { - if (StringUtils.equalsIgnoreCase(existingLink.getName(), link.getName())) { - doesNameExist = true; - } - } - if (!doesNameExist) { + // If a link of this name doesn't already exist in the list, add it + if (!list.stream().anyMatch((l -> StringUtils.equalsIgnoreCase(l.getName(), name)))) { super.add(link); } }