From 3c349cb70c775be1d13bbe1109d53bff1aa6e050 Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Mon, 6 Jul 2020 11:16:08 +0200 Subject: [PATCH 01/13] RIS, CSV and TSV live import implementation --- ...aratedImportMetadataSourceServiceImpl.java | 88 +++++++++++++ .../RisImportMetadataSourceServiceImpl.java | 117 ++++++++++++++++++ .../spring-dspace-addon-import-services.xml | 24 +++- .../config/spring/api/bibtex-integration.xml | 44 ++----- .../api/characterseparated-integration.xml | 80 ++++++++++++ .../spring/api/dublicore-metadata.mapper.xml | 59 +++++++++ dspace/config/spring/api/ris-integration.xml | 65 ++++++++++ 7 files changed, 441 insertions(+), 36 deletions(-) create mode 100644 dspace-api/src/main/java/org/dspace/importer/external/csv/service/CharacterSeparatedImportMetadataSourceServiceImpl.java create mode 100644 dspace-api/src/main/java/org/dspace/importer/external/ris/service/RisImportMetadataSourceServiceImpl.java create mode 100644 dspace/config/spring/api/characterseparated-integration.xml create mode 100644 dspace/config/spring/api/dublicore-metadata.mapper.xml create mode 100644 dspace/config/spring/api/ris-integration.xml diff --git a/dspace-api/src/main/java/org/dspace/importer/external/csv/service/CharacterSeparatedImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/csv/service/CharacterSeparatedImportMetadataSourceServiceImpl.java new file mode 100644 index 0000000000..75f2cb0776 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/csv/service/CharacterSeparatedImportMetadataSourceServiceImpl.java @@ -0,0 +1,88 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.importer.external.csv.service; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import au.com.bytecode.opencsv.CSVReader; +import org.dspace.importer.external.exception.FileSourceException; +import org.dspace.importer.external.metadatamapping.MetadataFieldConfig; +import org.dspace.importer.external.metadatamapping.contributor.MetadataContributor; +import org.dspace.importer.external.service.components.AbstractPlainMetadataSource; +import org.dspace.importer.external.service.components.dto.PlainMetadataKeyValueItem; +import org.dspace.importer.external.service.components.dto.PlainMetadataSourceDto; + + + +public class CharacterSeparatedImportMetadataSourceServiceImpl extends AbstractPlainMetadataSource { + + private String separator = ","; + + private String escapeCharacter = "\""; + + private String importSource = "CsvMetadataSource"; + + public void setSeparator(String separator) { + this.separator = separator; + } + + @Override + public String getImportSource() { + return importSource; + } + + public void setImportSource(String importSource) { + this.importSource = importSource; + } + + public void setEscapeCharacter(String escapeCharacter) { + this.escapeCharacter = escapeCharacter; + } + + @Override + protected List readData(InputStream inputStream) throws FileSourceException { + List plainMetadataList = new ArrayList<>(); + try (CSVReader csvReader = new CSVReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8), + separator.charAt(0), escapeCharacter.charAt(0));) { + List lines = csvReader.readAll(); + for (String [] items : lines) { + List keyValueList = new ArrayList<>(); + if (items != null) { + int size = items.length; + int count = 0; + while (count < size) { + PlainMetadataKeyValueItem keyValueItem = new PlainMetadataKeyValueItem(); + keyValueItem.setKey(String.valueOf(count)); + keyValueItem.setValue(items[count]); + keyValueList.add(keyValueItem); + count++; + } + PlainMetadataSourceDto dto = new PlainMetadataSourceDto(); + dto.setMetadata(keyValueList); + plainMetadataList.add(dto); + } + } + } catch (IOException e) { + throw new FileSourceException("Error reading file", e); + } + return plainMetadataList; + } + + @Override + public void setMetadataFieldMap(Map> metadataFieldMap) { + super.setMetadataFieldMap(metadataFieldMap); + } + +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/ris/service/RisImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/ris/service/RisImportMetadataSourceServiceImpl.java new file mode 100644 index 0000000000..73183242e1 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/ris/service/RisImportMetadataSourceServiceImpl.java @@ -0,0 +1,117 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.importer.external.ris.service; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.annotation.Resource; + +import org.dspace.importer.external.exception.FileSourceException; +import org.dspace.importer.external.service.components.AbstractPlainMetadataSource; +import org.dspace.importer.external.service.components.dto.PlainMetadataKeyValueItem; +import org.dspace.importer.external.service.components.dto.PlainMetadataSourceDto; + +/** + * Implements a metadata importer for RIS files + * Implementations insprider by BTE DataLoader {@link https://github.com/EKT/Biblio-Transformation-Engine/blob/master/bte-io/src/main/java/gr/ekt/bteio/loaders/RISDataLoader.java} + * + * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it) + */ +public class RisImportMetadataSourceServiceImpl extends AbstractPlainMetadataSource { + + @Override + public String getImportSource() { + return "RISMetadataSource"; + } + + protected List readData(InputStream inputStream) throws FileSourceException { + return aggreageteData(inputStream); + } + + private List aggreageteData(InputStream inputStream) throws FileSourceException { + List metadata = new ArrayList<>(); + List notAggregatedItems = notAggregatedData(inputStream); + List aggregatedTmpList = null; + Iterator itr = notAggregatedItems.iterator(); + while (itr.hasNext()) { + PlainMetadataKeyValueItem item = itr.next(); + if ("TY".equals(item.getKey())) { + if (aggregatedTmpList != null) { + PlainMetadataSourceDto dto = new PlainMetadataSourceDto(); + dto.setMetadata(new ArrayList<>(aggregatedTmpList)); + metadata.add(dto); + } + aggregatedTmpList = new ArrayList<>(); + aggregatedTmpList.add(item); + } else { + if (aggregatedTmpList != null) { + aggregatedTmpList.add(item); + // save last iteration metadata + if (!itr.hasNext()) { + PlainMetadataSourceDto dto = new PlainMetadataSourceDto(); + dto.setMetadata(new ArrayList<>(aggregatedTmpList)); + metadata.add(dto); + } + } + } + } + return metadata; + } + + private List notAggregatedData(InputStream inputStream) throws FileSourceException { + LinkedList items = new LinkedList<>(); + BufferedReader reader; + try { + reader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8")); + String line; + while ((line = reader.readLine()) != null) { + if (line.isEmpty() || line.equals("") || line.matches("^\\s*$")) { + continue; + } + //match valid RIS entry + Pattern risPattern = Pattern.compile("^([A-Z][A-Z0-9]) - (.*)$"); + Matcher risMatcher = risPattern.matcher(line); + if (risMatcher.matches()) { + PlainMetadataKeyValueItem keyValueItem = new PlainMetadataKeyValueItem(); + keyValueItem.setValue(risMatcher.group(2)); + keyValueItem.setKey(risMatcher.group(1)); + items.add(keyValueItem); + } else { + if (!items.isEmpty()) { + items.getLast().setValue(items.getLast().getValue().concat(line)); + } + } + } + } catch (Exception e) { + throw new FileSourceException("Cannot parse RIS file"); + } + return items; + } + + /** + * Retrieve the MetadataFieldMapping containing the mapping between RecordType + * (in this case PlainMetadataSourceDto.class) and Metadata + * + * @return The configured MetadataFieldMapping + */ + @Override + @SuppressWarnings("unchecked") + @Resource(name = "risMetadataFieldMap") + public void setMetadataFieldMap(@SuppressWarnings("rawtypes") Map metadataFieldMap) { + super.setMetadataFieldMap(metadataFieldMap); + } + +} diff --git a/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml b/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml index 3b4f7bad31..e356987787 100644 --- a/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml +++ b/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml @@ -23,8 +23,11 @@ lazy-init="false" autowire="byType" destroy-method="destroy"> - - + + + + + @@ -48,9 +51,24 @@ class="org.dspace.importer.external.pubmed.metadatamapping.PubmedFieldMapping"> + + + - + + + + + + + + + + diff --git a/dspace/config/spring/api/bibtex-integration.xml b/dspace/config/spring/api/bibtex-integration.xml index 9675ef82b3..eeabace1c7 100644 --- a/dspace/config/spring/api/bibtex-integration.xml +++ b/dspace/config/spring/api/bibtex-integration.xml @@ -17,58 +17,36 @@ only matters here for postprocessing of the value. The mapped MetadatumContributor has full control over what metadatafield is generated. - - - - - + + + + + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - + \ No newline at end of file diff --git a/dspace/config/spring/api/characterseparated-integration.xml b/dspace/config/spring/api/characterseparated-integration.xml new file mode 100644 index 0000000000..106d635671 --- /dev/null +++ b/dspace/config/spring/api/characterseparated-integration.xml @@ -0,0 +1,80 @@ + + + + + + + Defines which metadatum is mapped on which metadatum. Note that while the key must be unique it + only matters here for postprocessing of the value. The mapped MetadatumContributor has full control over + what metadatafield is generated. + + + + + + + + + + + + Defines which metadatum is mapped on which metadatum. Note that while the key must be unique it + only matters here for postprocessing of the value. The mapped MetadatumContributor has full control over + what metadatafield is generated. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dspace/config/spring/api/dublicore-metadata.mapper.xml b/dspace/config/spring/api/dublicore-metadata.mapper.xml new file mode 100644 index 0000000000..6461f129a5 --- /dev/null +++ b/dspace/config/spring/api/dublicore-metadata.mapper.xml @@ -0,0 +1,59 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dspace/config/spring/api/ris-integration.xml b/dspace/config/spring/api/ris-integration.xml new file mode 100644 index 0000000000..39b8edcf8d --- /dev/null +++ b/dspace/config/spring/api/ris-integration.xml @@ -0,0 +1,65 @@ + + + + + + + Defines which metadatum is mapped on which metadatum. Note that while the key must be unique it + only matters here for postprocessing of the value. The mapped MetadatumContributor has full control over + what metadatafield is generated. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From b7949a14d7b230d16ccd12139d3b28836ef7d24d Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Mon, 6 Jul 2020 16:35:48 +0200 Subject: [PATCH 02/13] Add endnote support and (partial) test --- ...ndnoteImportMetadataSourceServiceImpl.java | 99 +++++++++++ .../spring-dspace-addon-import-services.xml | 6 + .../rest/WorkspaceItemRestRepositoryIT.java | 165 +++++++++++++++++- .../app/rest/csv-missing-field-test.csv | 1 + .../org/dspace/app/rest/csv-test.csv | 1 + .../config/spring/api/endnote-integration.xml | 52 ++++++ 6 files changed, 323 insertions(+), 1 deletion(-) create mode 100644 dspace-api/src/main/java/org/dspace/importer/external/endnote/service/EndnoteImportMetadataSourceServiceImpl.java create mode 100644 dspace-server-webapp/src/test/resources/org/dspace/app/rest/csv-missing-field-test.csv create mode 100644 dspace-server-webapp/src/test/resources/org/dspace/app/rest/csv-test.csv create mode 100644 dspace/config/spring/api/endnote-integration.xml diff --git a/dspace-api/src/main/java/org/dspace/importer/external/endnote/service/EndnoteImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/endnote/service/EndnoteImportMetadataSourceServiceImpl.java new file mode 100644 index 0000000000..e0bc10f238 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/endnote/service/EndnoteImportMetadataSourceServiceImpl.java @@ -0,0 +1,99 @@ +package org.dspace.importer.external.endnote.service; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.dspace.importer.external.exception.FileSourceException; +import org.dspace.importer.external.metadatamapping.MetadataFieldConfig; +import org.dspace.importer.external.metadatamapping.contributor.MetadataContributor; +import org.dspace.importer.external.service.components.AbstractPlainMetadataSource; +import org.dspace.importer.external.service.components.dto.PlainMetadataKeyValueItem; +import org.dspace.importer.external.service.components.dto.PlainMetadataSourceDto; + +public class EndnoteImportMetadataSourceServiceImpl extends AbstractPlainMetadataSource { + + @Override + public String getImportSource() { + return "EndnoteMetadataSource"; + } + + @Override + protected List readData(InputStream fileInpuStream) throws FileSourceException { + List list = new ArrayList<>(); + try { + int lineForDebug = 3; + List tokenized = tokenize(fileInpuStream); + List tmpList = new ArrayList<>(); + for (PlainMetadataKeyValueItem item : tokenized) { + if (item.getKey() == null || item.getKey().isEmpty()) { + throw new FileSourceException("Null or empty key expected on line " + + lineForDebug + ". Keys cannot be null nor empty"); + } + if ("EF".equals(item.getKey())) { + break; + } + if ("ER".equals(item.getKey())) { + PlainMetadataSourceDto dto = new PlainMetadataSourceDto(); + dto.setMetadata(new ArrayList<>(tmpList)); + list.add(dto); + tmpList = new ArrayList<>(); + } else { + if (item.getValue() == null || item.getValue().isEmpty()) { + throw new FileSourceException("Null or empty value expected on line " + + lineForDebug + ". Value expected"); + } + tmpList.add(item); + } + lineForDebug++; + } + } catch (Exception e) { + throw new FileSourceException("Error reading file"); + } + return list; + } + + + private List tokenize(InputStream fileInpuStream) + throws IOException, FileSourceException { + BufferedReader reader = new BufferedReader(new InputStreamReader(fileInpuStream)); + String line; + line = reader.readLine(); + if (line == null || !line.startsWith("FN")) { + throw new FileSourceException("Invalid endNote file"); + } + line = reader.readLine(); + if (line == null || !line.startsWith("VR")) { + throw new FileSourceException("Invalid endNote file"); + } + Pattern pattern = Pattern.compile("(^[A-Z]{2}) ?(.*)$"); + List list = new ArrayList(); + while ((line = reader.readLine()) != null) { + line = line.trim(); + if (line.isEmpty() || line.equals("")) { + continue; + } + Matcher matcher = pattern.matcher(line); + if (matcher.matches()) { + PlainMetadataKeyValueItem item = new PlainMetadataKeyValueItem(); + item.setKey(matcher.group(1)); + item.setValue(matcher.group(2)); + list.add(item); + } + } + return list; + } + + @Override + public void setMetadataFieldMap(Map> metadataFieldMap) { + super.setMetadataFieldMap(metadataFieldMap); + } + +} diff --git a/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml b/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml index e356987787..dc988310f1 100644 --- a/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml +++ b/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml @@ -71,6 +71,12 @@ + + + + + diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java index c312faa831..f2a4b1bb99 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java @@ -871,7 +871,7 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration * * @throws Exception */ - public void createSingleWorkspaceItemFromFileWithOneEntryTest() throws Exception { + public void createSingleWorkspaceItemFromBibtexFileWithOneEntryTest() throws Exception { context.turnOffAuthorisationSystem(); //** GIVEN ** @@ -938,7 +938,170 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration bibtex.close(); } + @Test + /** + * Test the creation of workspaceitems POSTing to the resource collection endpoint a csv file + * + * @throws Exception + */ + public void createSingleWorkspaceItemFromCSVWithOneEntryTest() throws Exception { + context.turnOffAuthorisationSystem(); + //** GIVEN ** + //1. A community-collection structure with one parent community with sub-community and two collections. + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Community child1 = CommunityBuilder.createSubCommunity(context, parentCommunity) + .withName("Sub Community") + .build(); + Collection col1 = CollectionBuilder.createCollection(context, child1) + .withName("Collection 1") + .withSubmitterGroup(eperson) + .build(); + Collection col2 = CollectionBuilder.createCollection(context, child1) + .withName("Collection 2") + .withSubmitterGroup(eperson) + .build(); + + InputStream csv = getClass().getResourceAsStream("csv-test.csv"); + final MockMultipartFile csvFile = new MockMultipartFile("file", "/local/path/csv-test.csv", + "text/csv", csv); + + context.restoreAuthSystemState(); + + String authToken = getAuthToken(eperson.getEmail(), password); + // bulk create workspaceitems in the default collection (col1) + getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") + .file(csvFile)) + // bulk create should return 200, 201 (created) is better for single resource + .andExpect(status().isOk()) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.title'][0].value", + is("My Article"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.author'][0].value", + is("Nobody"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone" + + "['dc.date.issued'][0].value", + is("2006"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.source'][0].value", + is("My Journal"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone" + + "['dc.identifier.issn'][0].value", + is("Mock ISSN"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.type'][0].value", + is("Mock subtype"))) + .andExpect( + jsonPath("$._embedded.workspaceitems[0]._embedded.collection.id", is(col1.getID().toString()))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.upload.files[0]" + + ".metadata['dc.source'][0].value", + is("/local/path/csv-test.csv"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.upload.files[0]" + + ".metadata['dc.title'][0].value", + is("csv-test.csv"))) + .andExpect( + jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()); + ; + + // bulk create workspaceitems explicitly in the col2 + getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") + .file(csvFile) + .param("owningCollection", col2.getID().toString())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone" + + "['dc.title'][0].value", + is("My Article"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.author'][0].value", + is("Nobody"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone" + + "['dc.date.issued'][0].value", + is("2006"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.source'][0].value", + is("My Journal"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone" + + "['dc.identifier.issn'][0].value", + is("Mock ISSN"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.type'][0].value", + is("Mock subtype"))) + .andExpect( + jsonPath("$._embedded.workspaceitems[0]._embedded.collection.id", is(col2.getID().toString()))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.upload.files[0]" + + ".metadata['dc.source'][0].value", + is("/local/path/csv-test.csv"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.upload" + + ".files[0].metadata['dc.title'][0].value", + is("csv-test.csv"))) + .andExpect( + jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()); + csv.close(); + } + + @Test + /** + * Test the creation of workspaceitems POSTing to the resource collection endpoint a csv file + * with some missing data + * + * @throws Exception + */ + public void createSingleWorkspaceItemFromCSVWithOneEntryAndMissingDataTest() throws Exception { + context.turnOffAuthorisationSystem(); + + //** GIVEN ** + //1. A community-collection structure with one parent community with sub-community and two collections. + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Community child1 = CommunityBuilder.createSubCommunity(context, parentCommunity) + .withName("Sub Community") + .build(); + Collection col1 = CollectionBuilder.createCollection(context, child1) + .withName("Collection 1") + .withSubmitterGroup(eperson) + .build(); + Collection col2 = CollectionBuilder.createCollection(context, child1) + .withName("Collection 2") + .withSubmitterGroup(eperson) + .build(); + + InputStream csv = getClass().getResourceAsStream("bibtex-test.bib"); + final MockMultipartFile csvFile = new MockMultipartFile("file", "/local/path/csv-missing-field-test.csv", + "text/csv", csv); + + context.restoreAuthSystemState(); + + String authToken = getAuthToken(eperson.getEmail(), password); + + // bulk create workspaceitems in the default collection (col1) + getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") + .file(csvFile)) + // bulk create should return 200, 201 (created) is better for single resource + .andExpect(status().isOk()) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.title'][0].value", + is("My Article"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.author'][0].value", + is("Nobody"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone" + + "['dc.date.issued'][0].value").doesNotExist()) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.source'][0].value", + is("My Journal"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone" + + "['dc.identifier.issn'][0].value", + is("Mock ISSN"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.type'][0].value" + ).doesNotExist()) + .andExpect( + jsonPath("$._embedded.workspaceitems[0]._embedded.collection.id", is(col1.getID().toString()))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.upload.files[0]" + + ".metadata['dc.source'][0].value", + is("/local/path/csv-missing-field-test.csv"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.upload.files[0]" + + ".metadata['dc.title'][0].value", + is("csv-missing-field-test.csv"))) + .andExpect( + jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()); + ; + + csv.close(); + } @Test /** diff --git a/dspace-server-webapp/src/test/resources/org/dspace/app/rest/csv-missing-field-test.csv b/dspace-server-webapp/src/test/resources/org/dspace/app/rest/csv-missing-field-test.csv new file mode 100644 index 0000000000..58aecd1c8f --- /dev/null +++ b/dspace-server-webapp/src/test/resources/org/dspace/app/rest/csv-missing-field-test.csv @@ -0,0 +1 @@ +My article,Nobody,,My Journal,"This is my abstract, i user comma to check escape works fine",Mock ISSN \ No newline at end of file diff --git a/dspace-server-webapp/src/test/resources/org/dspace/app/rest/csv-test.csv b/dspace-server-webapp/src/test/resources/org/dspace/app/rest/csv-test.csv new file mode 100644 index 0000000000..d8bf63871c --- /dev/null +++ b/dspace-server-webapp/src/test/resources/org/dspace/app/rest/csv-test.csv @@ -0,0 +1 @@ +My article,Nobody,2006,My Journal,"This is my abstract, i user comma to check escape works fine",Mock ISSN,Mock subtype \ No newline at end of file diff --git a/dspace/config/spring/api/endnote-integration.xml b/dspace/config/spring/api/endnote-integration.xml new file mode 100644 index 0000000000..15ff3ca6f7 --- /dev/null +++ b/dspace/config/spring/api/endnote-integration.xml @@ -0,0 +1,52 @@ + + + + + + + Defines which metadatum is mapped on which metadatum. Note that while the key must be unique it + only matters here for postprocessing of the value. The mapped MetadatumContributor has full control over + what metadatafield is generated. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From dc958c9d670082987597932712f5b4203b56c658 Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Wed, 8 Jul 2020 00:06:49 +0200 Subject: [PATCH 03/13] Add support for multiple authors in *sv, complete implementation for RIS and Endnote, complete IT --- ...aratedImportMetadataSourceServiceImpl.java | 40 ++- .../EnhancedSimpleMetadataContributor.java | 94 ++++++ .../SimpleMetadataContributor.java | 7 + ...PubmedImportMetadataSourceServiceImpl.java | 12 + .../external/service/ImportService.java | 9 +- .../AbstractPlainMetadataSource.java | 11 + .../service/components/FileSource.java | 21 ++ .../spring-dspace-addon-import-services.xml | 36 ++- .../WorkspaceItemRestRepository.java | 5 +- .../rest/WorkspaceItemRestRepositoryIT.java | 278 +++++++++++++++++- .../app/rest/csv-missing-field-test.csv | 3 +- .../org/dspace/app/rest/csv-test.csv | 3 +- .../org/dspace/app/rest/endnote-test.enw | 10 + .../org/dspace/app/rest/ris-test.ris | 19 ++ .../app/rest/tsv-missing-field-test.tsv | 2 + .../org/dspace/app/rest/tsv-test.tsv | 2 + .../api/characterseparated-integration.xml | 6 +- 17 files changed, 524 insertions(+), 34 deletions(-) create mode 100644 dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/EnhancedSimpleMetadataContributor.java create mode 100644 dspace-server-webapp/src/test/resources/org/dspace/app/rest/endnote-test.enw create mode 100644 dspace-server-webapp/src/test/resources/org/dspace/app/rest/ris-test.ris create mode 100644 dspace-server-webapp/src/test/resources/org/dspace/app/rest/tsv-missing-field-test.tsv create mode 100644 dspace-server-webapp/src/test/resources/org/dspace/app/rest/tsv-test.tsv diff --git a/dspace-api/src/main/java/org/dspace/importer/external/csv/service/CharacterSeparatedImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/csv/service/CharacterSeparatedImportMetadataSourceServiceImpl.java index 75f2cb0776..3c1a838d7d 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/csv/service/CharacterSeparatedImportMetadataSourceServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/csv/service/CharacterSeparatedImportMetadataSourceServiceImpl.java @@ -27,14 +27,24 @@ import org.dspace.importer.external.service.components.dto.PlainMetadataSourceDt public class CharacterSeparatedImportMetadataSourceServiceImpl extends AbstractPlainMetadataSource { - private String separator = ","; + private char separator = ','; - private String escapeCharacter = "\""; + private char escapeCharacter = '"'; + + private Integer skipLines = 1; private String importSource = "CsvMetadataSource"; - public void setSeparator(String separator) { - this.separator = separator; + public void setSkipLines(Integer skipLines) { + this.skipLines = skipLines; + } + + public Integer getSkipLines() { + return skipLines; + } + + public void setSeparator(int separator) { + this.separator = (char)separator; } @Override @@ -46,32 +56,36 @@ public class CharacterSeparatedImportMetadataSourceServiceImpl extends AbstractP this.importSource = importSource; } - public void setEscapeCharacter(String escapeCharacter) { - this.escapeCharacter = escapeCharacter; + public void setEscapeCharacter(int escapeCharacter) { + this.escapeCharacter = (char)escapeCharacter; } @Override protected List readData(InputStream inputStream) throws FileSourceException { List plainMetadataList = new ArrayList<>(); try (CSVReader csvReader = new CSVReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8), - separator.charAt(0), escapeCharacter.charAt(0));) { + separator, escapeCharacter);) { List lines = csvReader.readAll(); - for (String [] items : lines) { + int listSize = lines == null ? 0 : lines.size(); + int count = skipLines; + while (count < listSize) { + String [] items = lines.get(count); List keyValueList = new ArrayList<>(); if (items != null) { int size = items.length; - int count = 0; - while (count < size) { + int index = 0; + while (index < size) { PlainMetadataKeyValueItem keyValueItem = new PlainMetadataKeyValueItem(); - keyValueItem.setKey(String.valueOf(count)); - keyValueItem.setValue(items[count]); + keyValueItem.setKey(String.valueOf(index)); + keyValueItem.setValue(items[index]); keyValueList.add(keyValueItem); - count++; + index++; } PlainMetadataSourceDto dto = new PlainMetadataSourceDto(); dto.setMetadata(keyValueList); plainMetadataList.add(dto); } + count++; } } catch (IOException e) { throw new FileSourceException("Error reading file", e); diff --git a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/EnhancedSimpleMetadataContributor.java b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/EnhancedSimpleMetadataContributor.java new file mode 100644 index 0000000000..0d2d009f7b --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/EnhancedSimpleMetadataContributor.java @@ -0,0 +1,94 @@ +package org.dspace.importer.external.metadatamapping.contributor; + +import java.io.IOException; +import java.io.StringReader; +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; + +import au.com.bytecode.opencsv.CSVReader; +import org.dspace.importer.external.metadatamapping.MetadatumDTO; +import org.dspace.importer.external.service.components.dto.PlainMetadataKeyValueItem; +import org.dspace.importer.external.service.components.dto.PlainMetadataSourceDto; + + +/** + * This class implements functionalities to handle common situation regarding plain metadata. + * In some scenario, like csv or tsv, the format don't allow lists. + * We can use this MetadataContribut to parse a given plain metadata and split it into + * related list, based on the delimiter. No escape character is present. + * Default values are comma (,) for delimiter, and double quote (") for escape character + * + * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it) + * + */ +public class EnhancedSimpleMetadataContributor extends SimpleMetadataContributor { + + private char delimiter = ','; + + private char escape = '"'; + + private boolean useEnhancer; + + public void setDelimiter(int delimiter) { + this.delimiter = (char)delimiter; + } + + public char getDelimiter() { + return delimiter; + } + + public void setEscape(int escape) { + this.escape = (char)escape; + } + + public char getEscape() { + return escape; + } + + public void setUseEnhancer(boolean useEnhancer) { + this.useEnhancer = useEnhancer; + } + + public boolean isUseEnhancer() { + return useEnhancer; + } + + @Override + public Collection contributeMetadata(PlainMetadataSourceDto t) { + Collection values = null; + if (!useEnhancer) { + values = super.contributeMetadata(t); + } else { + values = new LinkedList<>(); + for (PlainMetadataKeyValueItem metadatum : t.getMetadata()) { + if (getKey().equals(metadatum.getKey())) { + String[] splitted = splitToRecord(metadatum.getValue()); + for (String value : splitted) { + MetadatumDTO dcValue = new MetadatumDTO(); + dcValue.setValue(value); + dcValue.setElement(getField().getElement()); + dcValue.setQualifier(getField().getQualifier()); + dcValue.setSchema(getField().getSchema()); + values.add(dcValue); + } + } + } + } + return values; + } + + private String[] splitToRecord(String value) { + List rows; + try (CSVReader csvReader = new CSVReader(new StringReader(value), + delimiter, escape);) { + rows = csvReader.readAll(); + } catch (IOException e) { + //fallback, use the inpu as value + return new String[] { value }; + } + //must be one row + return rows.get(0); + } + +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/SimpleMetadataContributor.java b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/SimpleMetadataContributor.java index 21dd1bfcee..e61b3d2187 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/SimpleMetadataContributor.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/SimpleMetadataContributor.java @@ -91,4 +91,11 @@ public class SimpleMetadataContributor implements MetadataContributor supportedExtensions; + + public void setSupportedExtensions(List supportedExtensions) { + this.supportedExtensions = supportedExtensions; + } + + @Override + public List getSupportedExtensions() { + return supportedExtensions; + } + /** * Find the number of records matching a query; * @@ -430,4 +441,5 @@ public class PubmedImportMetadataSourceServiceImpl extends AbstractImportMetadat return records; } + } diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java b/dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java index 5a838e8027..35d8e59eb9 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java @@ -305,14 +305,17 @@ public class ImportService implements Destroyable { * @return a single record contains the metadatum * @throws FileMultipleOccurencesException if more than one entry is found */ - public ImportRecord getRecord(File file) throws FileMultipleOccurencesException, FileSourceException { + public ImportRecord getRecord(File file, String originalName) + throws FileMultipleOccurencesException, FileSourceException { ImportRecord importRecords = null; for (MetadataSource metadataSource : importSources.values()) { try (InputStream fileInputStream = new FileInputStream(file)) { if (metadataSource instanceof FileSource) { FileSource fileSource = (FileSource)metadataSource; - importRecords = fileSource.getRecord(fileInputStream); - break; + if (fileSource.isValidSourceForFile(originalName)) { + importRecords = fileSource.getRecord(fileInputStream); + break; + } } } catch (FileSourceException e) { log.debug(metadataSource.getImportSource() + " isn't a valid parser for file"); diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/components/AbstractPlainMetadataSource.java b/dspace-api/src/main/java/org/dspace/importer/external/service/components/AbstractPlainMetadataSource.java index 45b96f10b9..39f80cdd72 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/service/components/AbstractPlainMetadataSource.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/components/AbstractPlainMetadataSource.java @@ -33,6 +33,17 @@ public abstract class AbstractPlainMetadataSource extends AbstractMetadataFieldMapping implements FileSource { + private List supportedExtensions; + + public void setSupportedExtensions(List supportedExtensions) { + this.supportedExtensions = supportedExtensions; + } + + @Override + public List getSupportedExtensions() { + return supportedExtensions; + } + protected abstract List readData(InputStream fileInpuStream) throws FileSourceException; diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/components/FileSource.java b/dspace-api/src/main/java/org/dspace/importer/external/service/components/FileSource.java index ff05b8cdfb..16a3b0629d 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/service/components/FileSource.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/components/FileSource.java @@ -22,6 +22,8 @@ import org.dspace.importer.external.exception.FileSourceException; */ public interface FileSource extends MetadataSource { + public List getSupportedExtensions(); + /** * Return a list of ImportRecord constructed from input file. * @@ -43,4 +45,23 @@ public interface FileSource extends MetadataSource { public ImportRecord getRecord(InputStream inputStream) throws FileSourceException, FileMultipleOccurencesException; + + /** + * This method is used to decide if the FileSource manage the file format + * + * @param originalName the file file original name + * @return true if the FileSource can parse the file, false otherwise + */ + public default boolean isValidSourceForFile(String originalName) { + List extensions = getSupportedExtensions(); + if (extensions == null || extensions.isEmpty()) { + return false; + } + if (originalName != null && originalName.contains(".")) { + String extension = originalName.substring(originalName.lastIndexOf('.') + 1, + originalName.length()); + return getSupportedExtensions().contains(extension); + } + return false; + } } diff --git a/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml b/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml index dc988310f1..64627f68fc 100644 --- a/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml +++ b/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml @@ -44,6 +44,11 @@ class="org.dspace.importer.external.pubmed.service.PubmedImportMetadataSourceServiceImpl" scope="singleton"> + + + xml + + @@ -53,27 +58,56 @@ + + + ris + + + + + bib + bibtex + + + + + + csv + + - + + + + + tsv + + + + + enl + enw + + diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java index 3befc00a42..7ee157ab52 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java @@ -380,11 +380,14 @@ public class WorkspaceItemRestRepository extends DSpaceRestRepository - + + + + From 3f8e68055728405b11ca2dddf52810975b33f139 Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Wed, 8 Jul 2020 01:09:40 +0200 Subject: [PATCH 04/13] Add comments --- ...aratedImportMetadataSourceServiceImpl.java | 34 ++++++++++++++++++- ...ndnoteImportMetadataSourceServiceImpl.java | 12 +++++++ .../EnhancedSimpleMetadataContributor.java | 28 +++++++++++++++ .../SimpleMetadataContributor.java | 16 ++++++--- ...PubmedImportMetadataSourceServiceImpl.java | 6 ++++ .../AbstractPlainMetadataSource.java | 5 +++ .../service/components/FileSource.java | 3 ++ ...pper.xml => dublicore-metadata-mapper.xml} | 0 8 files changed, 99 insertions(+), 5 deletions(-) rename dspace/config/spring/api/{dublicore-metadata.mapper.xml => dublicore-metadata-mapper.xml} (100%) diff --git a/dspace-api/src/main/java/org/dspace/importer/external/csv/service/CharacterSeparatedImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/csv/service/CharacterSeparatedImportMetadataSourceServiceImpl.java index 3c1a838d7d..92002123e8 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/csv/service/CharacterSeparatedImportMetadataSourceServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/csv/service/CharacterSeparatedImportMetadataSourceServiceImpl.java @@ -20,11 +20,18 @@ import org.dspace.importer.external.exception.FileSourceException; import org.dspace.importer.external.metadatamapping.MetadataFieldConfig; import org.dspace.importer.external.metadatamapping.contributor.MetadataContributor; import org.dspace.importer.external.service.components.AbstractPlainMetadataSource; +import org.dspace.importer.external.service.components.MetadataSource; import org.dspace.importer.external.service.components.dto.PlainMetadataKeyValueItem; import org.dspace.importer.external.service.components.dto.PlainMetadataSourceDto; - +/** + * This class is an implementation of {@link MetadataSource} which extends {@link AbstractPlainMetadataSource} + * in order to parse "character separated" files like csv, tsv, etc using the Live Import framework. + * + * @author Pasquale Cavallo + * + */ public class CharacterSeparatedImportMetadataSourceServiceImpl extends AbstractPlainMetadataSource { private char separator = ','; @@ -35,14 +42,30 @@ public class CharacterSeparatedImportMetadataSourceServiceImpl extends AbstractP private String importSource = "CsvMetadataSource"; + /** + * Set the number of lines to skip at the start of the file. This method is suitable, + * for example, to skip file headers. + * + * @param skipLines number of the line at the start of the file to skip. + */ public void setSkipLines(Integer skipLines) { this.skipLines = skipLines; } + /** + * + * @return the number of the lines to skip + */ public Integer getSkipLines() { return skipLines; } + /** + * Method to inject the separator + * This must be the ASCII integer + * related to the char. + * In example, 9 for tab, 44 for comma + */ public void setSeparator(int separator) { this.separator = (char)separator; } @@ -52,10 +75,19 @@ public class CharacterSeparatedImportMetadataSourceServiceImpl extends AbstractP return importSource; } + /** + * Method to set the name of the source + */ public void setImportSource(String importSource) { this.importSource = importSource; } + /** + * Method to inject the escape character. This must be the ASCII integer + * related to the char. + * In example, 9 for tab, 44 for comma + * + */ public void setEscapeCharacter(int escapeCharacter) { this.escapeCharacter = (char)escapeCharacter; } diff --git a/dspace-api/src/main/java/org/dspace/importer/external/endnote/service/EndnoteImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/endnote/service/EndnoteImportMetadataSourceServiceImpl.java index e0bc10f238..995002e7ba 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/endnote/service/EndnoteImportMetadataSourceServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/endnote/service/EndnoteImportMetadataSourceServiceImpl.java @@ -1,3 +1,10 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ package org.dspace.importer.external.endnote.service; import java.io.BufferedReader; @@ -17,6 +24,11 @@ import org.dspace.importer.external.service.components.AbstractPlainMetadataSour import org.dspace.importer.external.service.components.dto.PlainMetadataKeyValueItem; import org.dspace.importer.external.service.components.dto.PlainMetadataSourceDto; +/** + * Implements a metadata importer for Endnote files + * + * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it) + */ public class EndnoteImportMetadataSourceServiceImpl extends AbstractPlainMetadataSource { @Override diff --git a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/EnhancedSimpleMetadataContributor.java b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/EnhancedSimpleMetadataContributor.java index 0d2d009f7b..19db700ff7 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/EnhancedSimpleMetadataContributor.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/EnhancedSimpleMetadataContributor.java @@ -30,26 +30,54 @@ public class EnhancedSimpleMetadataContributor extends SimpleMetadataContributor private boolean useEnhancer; + /** + * This method could be used to set the delimiter used during parse + * If no delimiter is set, comma will be used + */ public void setDelimiter(int delimiter) { this.delimiter = (char)delimiter; } + /** + * This method could be used to get the delimiter used in this class + */ public char getDelimiter() { return delimiter; } + /** + * Method to inject the escape character. + * This must be the ASCII integer + * related to the char. + * In example, 9 for tab, 44 for comma + * If no escape is set, double quote will be used + */ public void setEscape(int escape) { this.escape = (char)escape; } + /** + * Method to get the escape character. + * + */ public char getEscape() { return escape; } + /** + * Method to set up the enhancer. If set to false, enhancing will be not used + * In this case, the metadata value will + * As default, it is valued as false + * + */ public void setUseEnhancer(boolean useEnhancer) { this.useEnhancer = useEnhancer; } + /** + * + * @return true if the enhancer is set up, false otherwise. + */ public boolean isUseEnhancer() { return useEnhancer; } diff --git a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/SimpleMetadataContributor.java b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/SimpleMetadataContributor.java index e61b3d2187..1b9007f23c 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/SimpleMetadataContributor.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/SimpleMetadataContributor.java @@ -77,24 +77,32 @@ public class SimpleMetadataContributor implements MetadataContributor implements QuerySource, FileSource { @@ -54,6 +55,11 @@ public class PubmedImportMetadataSourceServiceImpl extends AbstractImportMetadat private List supportedExtensions; + /** + * Set the file extensions supported by this metadata service + * + * @param supportedExtensionsthe file extensions (xml,txt,...) supported by this service + */ public void setSupportedExtensions(List supportedExtensions) { this.supportedExtensions = supportedExtensions; } diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/components/AbstractPlainMetadataSource.java b/dspace-api/src/main/java/org/dspace/importer/external/service/components/AbstractPlainMetadataSource.java index 39f80cdd72..6c18ce20bc 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/service/components/AbstractPlainMetadataSource.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/components/AbstractPlainMetadataSource.java @@ -35,6 +35,11 @@ public abstract class AbstractPlainMetadataSource private List supportedExtensions; + /** + * Set the file extensions supported by the implementation of this abstract + * + * @param supportedExtensionsthe file extensions (xml,txt,...) supported by the specific implementation + */ public void setSupportedExtensions(List supportedExtensions) { this.supportedExtensions = supportedExtensions; } diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/components/FileSource.java b/dspace-api/src/main/java/org/dspace/importer/external/service/components/FileSource.java index 16a3b0629d..0d46da385b 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/service/components/FileSource.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/components/FileSource.java @@ -22,6 +22,9 @@ import org.dspace.importer.external.exception.FileSourceException; */ public interface FileSource extends MetadataSource { + /** + * Get the file extensions (xml, csv, txt, ...) supported by the FileSource + */ public List getSupportedExtensions(); /** diff --git a/dspace/config/spring/api/dublicore-metadata.mapper.xml b/dspace/config/spring/api/dublicore-metadata-mapper.xml similarity index 100% rename from dspace/config/spring/api/dublicore-metadata.mapper.xml rename to dspace/config/spring/api/dublicore-metadata-mapper.xml From 493ab0f496724c6000d17736a564f4be8824e9bb Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Wed, 8 Jul 2020 01:49:08 +0200 Subject: [PATCH 05/13] Add license header --- .../contributor/EnhancedSimpleMetadataContributor.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/EnhancedSimpleMetadataContributor.java b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/EnhancedSimpleMetadataContributor.java index 19db700ff7..93f59b8efd 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/EnhancedSimpleMetadataContributor.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/EnhancedSimpleMetadataContributor.java @@ -1,3 +1,10 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ package org.dspace.importer.external.metadatamapping.contributor; import java.io.IOException; From 6db52f37fd8d8ac4b5be073cd7015c9a26840673 Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Wed, 12 Aug 2020 15:15:49 +0200 Subject: [PATCH 06/13] Add javadoc and comment --- ...aratedImportMetadataSourceServiceImpl.java | 20 ++++++++++++++ ...ndnoteImportMetadataSourceServiceImpl.java | 22 +++++++++++++++ .../RisImportMetadataSourceServiceImpl.java | 27 +++++++++++++++++-- .../AbstractPlainMetadataSource.java | 16 ----------- .../service/components/FileSource.java | 5 ---- 5 files changed, 67 insertions(+), 23 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/importer/external/csv/service/CharacterSeparatedImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/csv/service/CharacterSeparatedImportMetadataSourceServiceImpl.java index 92002123e8..2fc23f3902 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/csv/service/CharacterSeparatedImportMetadataSourceServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/csv/service/CharacterSeparatedImportMetadataSourceServiceImpl.java @@ -92,27 +92,47 @@ public class CharacterSeparatedImportMetadataSourceServiceImpl extends AbstractP this.escapeCharacter = (char)escapeCharacter; } + /** + * The method process any kind of "character separated" files, like CSV, TSV, and so on. + * It return a List of PlainMetadataSourceDto. + * Using the superclass methods AbstractPlainMetadataSource.getRecord(s), any of this + * element will then be converted in an {@link org.dspace.importer.external.datamodel.ImportRecord}. + + * Columns will be identified by their position, zero based notation. + * Separator character and escape character MUST be defined at class level. Number of lines to skip (headers) + * could also be defined in the field skipLines. + * + * @param InputStream The inputStream of the file + * @return A list of PlainMetadataSourceDto + * @throws FileSourceException if, for any reason, the file is not parsable + + */ @Override protected List readData(InputStream inputStream) throws FileSourceException { List plainMetadataList = new ArrayList<>(); try (CSVReader csvReader = new CSVReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8), separator, escapeCharacter);) { + // read all row List lines = csvReader.readAll(); int listSize = lines == null ? 0 : lines.size(); int count = skipLines; + // iterate over row (skipping the first skipLines) while (count < listSize) { String [] items = lines.get(count); List keyValueList = new ArrayList<>(); if (items != null) { int size = items.length; int index = 0; + //iterate over column in the selected row while (index < size) { + //create key/value item for the specifics row/column PlainMetadataKeyValueItem keyValueItem = new PlainMetadataKeyValueItem(); keyValueItem.setKey(String.valueOf(index)); keyValueItem.setValue(items[index]); keyValueList.add(keyValueItem); index++; } + //save all column key/value for the given row PlainMetadataSourceDto dto = new PlainMetadataSourceDto(); dto.setMetadata(keyValueList); plainMetadataList.add(dto); diff --git a/dspace-api/src/main/java/org/dspace/importer/external/endnote/service/EndnoteImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/endnote/service/EndnoteImportMetadataSourceServiceImpl.java index 995002e7ba..a6e9ece160 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/endnote/service/EndnoteImportMetadataSourceServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/endnote/service/EndnoteImportMetadataSourceServiceImpl.java @@ -36,13 +36,25 @@ public class EndnoteImportMetadataSourceServiceImpl extends AbstractPlainMetadat return "EndnoteMetadataSource"; } + /** + * This method map the data present in the inputStream, then return a list PlainMetadataSourceDto. + * Any PlainMetadataSourceDto will be used to create a single {@link org.dspace.importer.external.datamodel.ImportRecord} + * + * @param inputStream the inputStream of the Endnote file + * @return List of {@link org.dspace.importer.external.service.components.dto.PlainMetadataSourceDto} + * @throws FileSourceException + * @see org.dspace.importer.external.service.components.AbstractPlainMetadataSource + */ @Override protected List readData(InputStream fileInpuStream) throws FileSourceException { List list = new ArrayList<>(); try { + // row start from 3, because the first 2 (FN and VR) will be removed by tokenize int lineForDebug = 3; List tokenized = tokenize(fileInpuStream); List tmpList = new ArrayList<>(); + //iterate over key/value pairs, create a new PlainMetadataSourceDto on "ER" rows (which means "new record) + // and stop on EF (end of file). for (PlainMetadataKeyValueItem item : tokenized) { if (item.getKey() == null || item.getKey().isEmpty()) { throw new FileSourceException("Null or empty key expected on line " @@ -72,6 +84,16 @@ public class EndnoteImportMetadataSourceServiceImpl extends AbstractPlainMetadat } + /** + * This method iterate over file rows, split content in a list of key/value items through RexExp + * and save the content sequentially. + * Key "FN" and "VR", which is a preamble in Endnote, will be checked but not saved. + * + * @param fileInpuStream the inputStream of the Endnote file + * @return A list of key/value items which map the file's row sequentially + * @throws IOException + * @throws FileSourceException + */ private List tokenize(InputStream fileInpuStream) throws IOException, FileSourceException { BufferedReader reader = new BufferedReader(new InputStreamReader(fileInpuStream)); diff --git a/dspace-api/src/main/java/org/dspace/importer/external/ris/service/RisImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/ris/service/RisImportMetadataSourceServiceImpl.java index 73183242e1..1fb5796497 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/ris/service/RisImportMetadataSourceServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/ris/service/RisImportMetadataSourceServiceImpl.java @@ -38,14 +38,28 @@ public class RisImportMetadataSourceServiceImpl extends AbstractPlainMetadataSou } protected List readData(InputStream inputStream) throws FileSourceException { - return aggreageteData(inputStream); + return aggregateData(inputStream); } - private List aggreageteData(InputStream inputStream) throws FileSourceException { + /** + * This method map the data present in the inputStream, then return a list PlainMetadataSourceDto. + * Any PlainMetadataSourceDto will be used to create a single {@link org.dspace.importer.external.datamodel.ImportRecord} + * + * @see org.dspace.importer.external.service.components.AbstractPlainMetadataSource + * + * @param inputStream the inputStream of the RIS file + * @return List of {@link org.dspace.importer.external.service.components.dto.PlainMetadataSourceDto} + * @throws FileSourceException + */ + private List aggregateData(InputStream inputStream) throws FileSourceException { List metadata = new ArrayList<>(); + //map any line of the field to a key/value pair List notAggregatedItems = notAggregatedData(inputStream); List aggregatedTmpList = null; Iterator itr = notAggregatedItems.iterator(); + // iterate over the list of key/value items + // create a new PlainMetadataSourceDto (which map and ImportRecord) + // any times the key is "TY" (content separator in RIS) while (itr.hasNext()) { PlainMetadataKeyValueItem item = itr.next(); if ("TY".equals(item.getKey())) { @@ -71,6 +85,15 @@ public class RisImportMetadataSourceServiceImpl extends AbstractPlainMetadataSou return metadata; } + /** + * This method transform any row of the RIS file into a PlainMetadataKeyValueItem, + * splitting the row sequentially through a RegExp without take care of the means of the data. + * In this way, all entries present in the file are mapped in the resulting list. + * + * @param inputStream the inputStrem of the file + * @return A list + * @throws FileSourceException + */ private List notAggregatedData(InputStream inputStream) throws FileSourceException { LinkedList items = new LinkedList<>(); BufferedReader reader; diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/components/AbstractPlainMetadataSource.java b/dspace-api/src/main/java/org/dspace/importer/external/service/components/AbstractPlainMetadataSource.java index 66a9e14623..019cf33177 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/service/components/AbstractPlainMetadataSource.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/components/AbstractPlainMetadataSource.java @@ -33,22 +33,6 @@ public abstract class AbstractPlainMetadataSource extends AbstractMetadataFieldMapping implements FileSource { - private List supportedExtensions; - - /** - * Set the file extensions supported by the implementation of this abstract - * - * @param supportedExtensionsthe file extensions (xml,txt,...) supported by the specific implementation - */ - public void setSupportedExtensions(List supportedExtensions) { - this.supportedExtensions = supportedExtensions; - } - - @Override - public List getSupportedExtensions() { - return supportedExtensions; - } - protected abstract List readData(InputStream fileInpuStream) throws FileSourceException; diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/components/FileSource.java b/dspace-api/src/main/java/org/dspace/importer/external/service/components/FileSource.java index 73cb4a1ba8..5bef0984df 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/service/components/FileSource.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/components/FileSource.java @@ -67,9 +67,4 @@ public interface FileSource extends MetadataSource { return false; } - /** - * Get the file extensions (xml, csv, txt, ...) supported by the FileSource implementation - */ - public List getSupportedExtensions(); - } From 01dac23829c237ecba2e478a3f734554aa4e8da7 Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Thu, 13 Aug 2020 15:19:18 +0200 Subject: [PATCH 07/13] Javadoc and minor changes --- .../EnhancedSimpleMetadataContributor.java | 46 +++++-------------- .../RisImportMetadataSourceServiceImpl.java | 3 +- .../spring-dspace-addon-import-services.xml | 41 +++++++++-------- .../api/characterseparated-integration.xml | 1 - 4 files changed, 34 insertions(+), 57 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/EnhancedSimpleMetadataContributor.java b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/EnhancedSimpleMetadataContributor.java index 93f59b8efd..53ac461f83 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/EnhancedSimpleMetadataContributor.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/EnhancedSimpleMetadataContributor.java @@ -35,8 +35,6 @@ public class EnhancedSimpleMetadataContributor extends SimpleMetadataContributor private char escape = '"'; - private boolean useEnhancer; - /** * This method could be used to set the delimiter used during parse * If no delimiter is set, comma will be used @@ -71,42 +69,20 @@ public class EnhancedSimpleMetadataContributor extends SimpleMetadataContributor return escape; } - /** - * Method to set up the enhancer. If set to false, enhancing will be not used - * In this case, the metadata value will - * As default, it is valued as false - * - */ - public void setUseEnhancer(boolean useEnhancer) { - this.useEnhancer = useEnhancer; - } - - /** - * - * @return true if the enhancer is set up, false otherwise. - */ - public boolean isUseEnhancer() { - return useEnhancer; - } - @Override public Collection contributeMetadata(PlainMetadataSourceDto t) { Collection values = null; - if (!useEnhancer) { - values = super.contributeMetadata(t); - } else { - values = new LinkedList<>(); - for (PlainMetadataKeyValueItem metadatum : t.getMetadata()) { - if (getKey().equals(metadatum.getKey())) { - String[] splitted = splitToRecord(metadatum.getValue()); - for (String value : splitted) { - MetadatumDTO dcValue = new MetadatumDTO(); - dcValue.setValue(value); - dcValue.setElement(getField().getElement()); - dcValue.setQualifier(getField().getQualifier()); - dcValue.setSchema(getField().getSchema()); - values.add(dcValue); - } + values = new LinkedList<>(); + for (PlainMetadataKeyValueItem metadatum : t.getMetadata()) { + if (getKey().equals(metadatum.getKey())) { + String[] splitted = splitToRecord(metadatum.getValue()); + for (String value : splitted) { + MetadatumDTO dcValue = new MetadatumDTO(); + dcValue.setValue(value); + dcValue.setElement(getField().getElement()); + dcValue.setQualifier(getField().getQualifier()); + dcValue.setSchema(getField().getSchema()); + values.add(dcValue); } } } diff --git a/dspace-api/src/main/java/org/dspace/importer/external/ris/service/RisImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/ris/service/RisImportMetadataSourceServiceImpl.java index 1fb5796497..2574e187df 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/ris/service/RisImportMetadataSourceServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/ris/service/RisImportMetadataSourceServiceImpl.java @@ -37,6 +37,7 @@ public class RisImportMetadataSourceServiceImpl extends AbstractPlainMetadataSou return "RISMetadataSource"; } + @Override protected List readData(InputStream inputStream) throws FileSourceException { return aggregateData(inputStream); } @@ -119,7 +120,7 @@ public class RisImportMetadataSourceServiceImpl extends AbstractPlainMetadataSou } } } catch (Exception e) { - throw new FileSourceException("Cannot parse RIS file"); + throw new FileSourceException("Cannot parse RIS file", e); } return items; } diff --git a/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml b/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml index 0156ff50d7..e2e670cc70 100644 --- a/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml +++ b/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml @@ -19,15 +19,16 @@ - - - - - - + + + + + + @@ -40,9 +41,9 @@ Omitting this property will default to searching over all configured ImportService implementations --> - - + + @@ -52,12 +53,12 @@ - - + ris @@ -65,8 +66,8 @@ - + bib @@ -75,8 +76,8 @@ - + @@ -86,8 +87,8 @@ - + @@ -99,8 +100,8 @@ - + diff --git a/dspace/config/spring/api/characterseparated-integration.xml b/dspace/config/spring/api/characterseparated-integration.xml index ef1d84b70e..5057acaff6 100644 --- a/dspace/config/spring/api/characterseparated-integration.xml +++ b/dspace/config/spring/api/characterseparated-integration.xml @@ -53,7 +53,6 @@ - From 9051b54259c0e3751d35d61d0f2d17d9a139058c Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Thu, 13 Aug 2020 15:23:04 +0200 Subject: [PATCH 08/13] add exception to logger --- .../endnote/service/EndnoteImportMetadataSourceServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspace-api/src/main/java/org/dspace/importer/external/endnote/service/EndnoteImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/endnote/service/EndnoteImportMetadataSourceServiceImpl.java index a6e9ece160..6e613194dd 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/endnote/service/EndnoteImportMetadataSourceServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/endnote/service/EndnoteImportMetadataSourceServiceImpl.java @@ -78,7 +78,7 @@ public class EndnoteImportMetadataSourceServiceImpl extends AbstractPlainMetadat lineForDebug++; } } catch (Exception e) { - throw new FileSourceException("Error reading file"); + throw new FileSourceException("Error reading file", e); } return list; } From 5a8311189b5dea7fbcbb00209fe8538d5705db67 Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Thu, 13 Aug 2020 15:30:47 +0200 Subject: [PATCH 09/13] add comments in endnote --- .../service/EndnoteImportMetadataSourceServiceImpl.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dspace-api/src/main/java/org/dspace/importer/external/endnote/service/EndnoteImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/endnote/service/EndnoteImportMetadataSourceServiceImpl.java index 6e613194dd..9881832369 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/endnote/service/EndnoteImportMetadataSourceServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/endnote/service/EndnoteImportMetadataSourceServiceImpl.java @@ -53,7 +53,7 @@ public class EndnoteImportMetadataSourceServiceImpl extends AbstractPlainMetadat int lineForDebug = 3; List tokenized = tokenize(fileInpuStream); List tmpList = new ArrayList<>(); - //iterate over key/value pairs, create a new PlainMetadataSourceDto on "ER" rows (which means "new record) + // iterate over key/value pairs, create a new PlainMetadataSourceDto on "ER" rows (which means "new record) // and stop on EF (end of file). for (PlainMetadataKeyValueItem item : tokenized) { if (item.getKey() == null || item.getKey().isEmpty()) { @@ -61,9 +61,12 @@ public class EndnoteImportMetadataSourceServiceImpl extends AbstractPlainMetadat + lineForDebug + ". Keys cannot be null nor empty"); } if ("EF".equals(item.getKey())) { + // end of file break; } if ("ER".equals(item.getKey())) { + // new ImportRecord start from here (ER is a content delimiter) + // save the previous, then create a new one PlainMetadataSourceDto dto = new PlainMetadataSourceDto(); dto.setMetadata(new ArrayList<>(tmpList)); list.add(dto); @@ -99,6 +102,7 @@ public class EndnoteImportMetadataSourceServiceImpl extends AbstractPlainMetadat BufferedReader reader = new BufferedReader(new InputStreamReader(fileInpuStream)); String line; line = reader.readLine(); + // FN and VR works as preamble, just check and skip them if (line == null || !line.startsWith("FN")) { throw new FileSourceException("Invalid endNote file"); } @@ -106,10 +110,13 @@ public class EndnoteImportMetadataSourceServiceImpl extends AbstractPlainMetadat if (line == null || !line.startsWith("VR")) { throw new FileSourceException("Invalid endNote file"); } + // split any row into first part ^[A-Z]{2} used as key (the meaning of the data) + // and second part ?(.*) used as value (the data) Pattern pattern = Pattern.compile("(^[A-Z]{2}) ?(.*)$"); List list = new ArrayList(); while ((line = reader.readLine()) != null) { line = line.trim(); + // skip empty lines if (line.isEmpty() || line.equals("")) { continue; } From 21a31a910bed0219013c3615e9a0d94baccdcabf Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Thu, 13 Aug 2020 16:30:43 +0200 Subject: [PATCH 10/13] Update tests --- .../rest/WorkspaceItemRestRepositoryIT.java | 210 +++++++++++++++--- 1 file changed, 174 insertions(+), 36 deletions(-) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java index 63d5381257..e6e70ecdaa 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java @@ -897,9 +897,11 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration context.restoreAuthSystemState(); + AtomicReference> idRef = new AtomicReference<>(); String authToken = getAuthToken(eperson.getEmail(), password); - // create a workspaceitem from a single bibliographic entry file explicitly in the default collection (col1) - getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") + try { + // create a workspaceitem from a single bibliographic entry file explicitly in the default collection (col1) + getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(bibtexFile)) // create should return 200, 201 (created) is better for single resource .andExpect(status().isOk()) @@ -914,11 +916,20 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration + ".metadata['dc.title'][0].value", is("bibtex-test.bib"))) .andExpect( - jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()); - ; + jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()) + .andDo(result -> idRef.set(read(result.getResponse().getContentAsString(), + "$._embedded.workspaceitems[*].id"))); + } finally { + if (idRef != null && idRef.get() != null) { + for (int i : idRef.get()) { + WorkspaceItemBuilder.deleteWorkspaceItem(i); + } + } + } // create a workspaceitem from a single bibliographic entry file explicitly in the col2 - getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") + try { + getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(bibtexFile) .param("owningCollection", col2.getID().toString())) .andExpect(status().isOk()) @@ -933,8 +944,16 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration + ".files[0].metadata['dc.title'][0].value", is("bibtex-test.bib"))) .andExpect( - jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()); - + jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()) + .andDo(result -> idRef.set(read(result.getResponse().getContentAsString(), + "$._embedded.workspaceitems[*].id"))); + } finally { + if (idRef != null && idRef.get() != null) { + for (int i : idRef.get()) { + WorkspaceItemBuilder.deleteWorkspaceItem(i); + } + } + } bibtex.close(); } @@ -972,7 +991,9 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration String authToken = getAuthToken(eperson.getEmail(), password); // bulk create workspaceitems in the default collection (col1) - getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") + AtomicReference> idRef = new AtomicReference<>(); + try { + getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(csvFile)) // bulk create should return 200, 201 (created) is better for single resource .andExpect(status().isOk()) @@ -998,11 +1019,20 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration + ".metadata['dc.title'][0].value", is("csv-test.csv"))) .andExpect( - jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()); - ; + jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()) + .andDo(result -> idRef.set(read(result.getResponse().getContentAsString(), + "$._embedded.workspaceitems[*].id"))); + } finally { + if (idRef != null && idRef.get() != null) { + for (int i : idRef.get()) { + WorkspaceItemBuilder.deleteWorkspaceItem(i); + } + } + } // bulk create workspaceitems explicitly in the col2 - getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") + try { + getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(csvFile) .param("owningCollection", col2.getID().toString())) .andExpect(status().isOk()) @@ -1029,7 +1059,16 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration + ".files[0].metadata['dc.title'][0].value", is("csv-test.csv"))) .andExpect( - jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()); + jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()) + .andDo(result -> idRef.set(read(result.getResponse().getContentAsString(), + "$._embedded.workspaceitems[*].id"))); + } finally { + if (idRef != null && idRef.get() != null) { + for (int i : idRef.get()) { + WorkspaceItemBuilder.deleteWorkspaceItem(i); + } + } + } csv.close(); } @@ -1067,9 +1106,11 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration context.restoreAuthSystemState(); String authToken = getAuthToken(eperson.getEmail(), password); - + AtomicReference> idRef = new AtomicReference<>(); // bulk create workspaceitems in the default collection (col1) - getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") + + try { + getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(csvFile)) // bulk create should return 200, 201 (created) is better for single resource .andExpect(status().isOk()) @@ -1097,8 +1138,17 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration + ".metadata['dc.title'][0].value", is("csv-missing-field-test.csv"))) .andExpect( - jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()); - csv.close(); + jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()) + .andDo(result -> idRef.set(read(result.getResponse().getContentAsString(), + "$._embedded.workspaceitems[*].id"))); + } finally { + if (idRef != null && idRef.get() != null) { + for (int i : idRef.get()) { + WorkspaceItemBuilder.deleteWorkspaceItem(i); + } + } + } + csv.close(); } @Test @@ -1134,8 +1184,11 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration context.restoreAuthSystemState(); String authToken = getAuthToken(eperson.getEmail(), password); + AtomicReference> idRef = new AtomicReference<>(); + // bulk create workspaceitems in the default collection (col1) - getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") + try { + getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(tsvFile)) // bulk create should return 200, 201 (created) is better for single resource .andExpect(status().isOk()) @@ -1161,8 +1214,16 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration + ".metadata['dc.title'][0].value", is("tsv-test.tsv"))) .andExpect( - jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()); - ; + jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()) + .andDo(result -> idRef.set(read(result.getResponse().getContentAsString(), + "$._embedded.workspaceitems[*].id"))); + } finally { + if (idRef != null && idRef.get() != null) { + for (int i : idRef.get()) { + WorkspaceItemBuilder.deleteWorkspaceItem(i); + } + } + } tsv.close(); } @@ -1194,8 +1255,11 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration context.restoreAuthSystemState(); String authToken = getAuthToken(eperson.getEmail(), password); + AtomicReference> idRef = new AtomicReference<>(); + // bulk create workspaceitems in the default collection (col1) - getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") + try { + getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(tsvFile)) // bulk create should return 200, 201 (created) is better for single resource .andExpect(status().isOk()) @@ -1221,8 +1285,16 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration + ".metadata['dc.title'][0].value", is("ris-test.ris"))) .andExpect( - jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()); - ; + jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()) + .andDo(result -> idRef.set(read(result.getResponse().getContentAsString(), + "$._embedded.workspaceitems[*].id"))); + } finally { + if (idRef != null && idRef.get() != null) { + for (int i : idRef.get()) { + WorkspaceItemBuilder.deleteWorkspaceItem(i); + } + } + } ris.close(); } @@ -1253,8 +1325,10 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration context.restoreAuthSystemState(); String authToken = getAuthToken(eperson.getEmail(), password); + AtomicReference> idRef = new AtomicReference<>(); // bulk create workspaceitems in the default collection (col1) - getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") + try { + getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(endnoteFile)) // bulk create should return 200, 201 (created) is better for single resource .andExpect(status().isOk()) @@ -1281,8 +1355,16 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration + ".metadata['dc.title'][0].value", is("endnote-test.enw"))) .andExpect( - jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()); - ; + jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()) + .andDo(result -> idRef.set(read(result.getResponse().getContentAsString(), + "$._embedded.workspaceitems[*].id"))); + } finally { + if (idRef != null && idRef.get() != null) { + for (int i : idRef.get()) { + WorkspaceItemBuilder.deleteWorkspaceItem(i); + } + } + } endnote.close(); } @@ -1321,9 +1403,11 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration context.restoreAuthSystemState(); String authToken = getAuthToken(eperson.getEmail(), password); + AtomicReference> idRef = new AtomicReference<>(); // bulk create workspaceitems in the default collection (col1) - getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") + try { + getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(csvFile)) // bulk create should return 200, 201 (created) is better for single resource .andExpect(status().isOk()) @@ -1351,7 +1435,16 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration + ".metadata['dc.title'][0].value", is("tsv-missing-field-test.tsv"))) .andExpect( - jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()); + jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist()) + .andDo(result -> idRef.set(read(result.getResponse().getContentAsString(), + "$._embedded.workspaceitems[*].id"))); + } finally { + if (idRef != null && idRef.get() != null) { + for (int i : idRef.get()) { + WorkspaceItemBuilder.deleteWorkspaceItem(i); + } + } + } tsv.close(); } @@ -1391,8 +1484,11 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration context.restoreAuthSystemState(); String authToken = getAuthToken(eperson.getEmail(), password); + AtomicReference> idRef = new AtomicReference<>(); + // create a workspaceitem from a single bibliographic entry file explicitly in the default collection (col1) - getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") + try { + getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(bibtexFile).file(pubmedFile)) // create should return 200, 201 (created) is better for single resource .andExpect(status().isOk()) @@ -1413,10 +1509,20 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration is("/local/path/pubmed-test.xml"))) .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.upload.files[1]" + ".metadata['dc.title'][0].value", - is("pubmed-test.xml"))); + is("pubmed-test.xml"))) + .andDo(result -> idRef.set(read(result.getResponse().getContentAsString(), + "$._embedded.workspaceitems[*].id"))); + } finally { + if (idRef != null && idRef.get() != null) { + for (int i : idRef.get()) { + WorkspaceItemBuilder.deleteWorkspaceItem(i); + } + } + } // create a workspaceitem from a single bibliographic entry file explicitly in the col2 - getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") + try { + getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(bibtexFile).file(pubmedFile) .param("owningCollection", col2.getID().toString())) .andExpect(status().isOk()) @@ -1437,7 +1543,16 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration is("/local/path/pubmed-test.xml"))) .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.upload.files[1]" + ".metadata['dc.title'][0].value", - is("pubmed-test.xml"))); + is("pubmed-test.xml"))) + .andDo(result -> idRef.set(read(result.getResponse().getContentAsString(), + "$._embedded.workspaceitems[*].id"))); + } finally { + if (idRef != null && idRef.get() != null) { + for (int i : idRef.get()) { + WorkspaceItemBuilder.deleteWorkspaceItem(i); + } + } + } bibtex.close(); xmlIS.close(); } @@ -1477,6 +1592,7 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration context.restoreAuthSystemState(); String authToken = getAuthToken(eperson.getEmail(), password); + // create a workspaceitem from a single bibliographic entry file explicitly in the default collection (col1) getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(bibtexFile)) @@ -1519,8 +1635,11 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration context.restoreAuthSystemState(); String authToken = getAuthToken(eperson.getEmail(), password); + AtomicReference> idRef = new AtomicReference<>(); + // create a workspaceitem from a single bibliographic entry file explicitly in the default collection (col1) - getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") + try { + getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(pubmedFile)) .andExpect(status().isOk()) .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.title'][0].value", @@ -1537,10 +1656,21 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration is("/local/path/pubmed-test.xml"))) .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.upload.files[0]" + ".metadata['dc.title'][0].value", - is("pubmed-test.xml"))); + is("pubmed-test.xml"))) + .andDo(result -> idRef.set(read(result.getResponse().getContentAsString(), + "$._embedded.workspaceitems[*].id"))); + } finally { + if (idRef != null && idRef.get() != null) { + for (int i : idRef.get()) { + WorkspaceItemBuilder.deleteWorkspaceItem(i); + } + } + } + // create a workspaceitem from a single bibliographic entry file explicitly in the col2 - getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") + try { + getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(pubmedFile) .param("owningCollection", col2.getID().toString())) .andExpect(status().isOk()) @@ -1556,8 +1686,16 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.upload.files[0].metadata['dc.source'][0].value", is("/local/path/pubmed-test.xml"))) .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.upload.files[0].metadata['dc.title'][0].value", - is("pubmed-test.xml"))); - + is("pubmed-test.xml"))) + .andDo(result -> idRef.set(read(result.getResponse().getContentAsString(), + "$._embedded.workspaceitems[*].id"))); + } finally { + if (idRef != null && idRef.get() != null) { + for (int i : idRef.get()) { + WorkspaceItemBuilder.deleteWorkspaceItem(i); + } + } + } xmlIS.close(); } From 55901fe1b9f8a8208f540f7ec14d1f8854dc15b5 Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Fri, 14 Aug 2020 11:20:15 +0200 Subject: [PATCH 11/13] Use unicode in char separated files import --- ...acterSeparatedImportMetadataSourceServiceImpl.java | 8 ++++---- .../EnhancedSimpleMetadataContributor.java | 11 +++++++---- .../spring/spring-dspace-addon-import-services.xml | 2 +- .../spring/api/characterseparated-integration.xml | 3 --- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/importer/external/csv/service/CharacterSeparatedImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/csv/service/CharacterSeparatedImportMetadataSourceServiceImpl.java index 2fc23f3902..31ee1e5e5a 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/csv/service/CharacterSeparatedImportMetadataSourceServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/csv/service/CharacterSeparatedImportMetadataSourceServiceImpl.java @@ -66,8 +66,8 @@ public class CharacterSeparatedImportMetadataSourceServiceImpl extends AbstractP * related to the char. * In example, 9 for tab, 44 for comma */ - public void setSeparator(int separator) { - this.separator = (char)separator; + public void setSeparator(char separator) { + this.separator = separator; } @Override @@ -88,8 +88,8 @@ public class CharacterSeparatedImportMetadataSourceServiceImpl extends AbstractP * In example, 9 for tab, 44 for comma * */ - public void setEscapeCharacter(int escapeCharacter) { - this.escapeCharacter = (char)escapeCharacter; + public void setEscapeCharacter(char escapeCharacter) { + this.escapeCharacter = escapeCharacter; } /** diff --git a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/EnhancedSimpleMetadataContributor.java b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/EnhancedSimpleMetadataContributor.java index 53ac461f83..b06322ac2c 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/EnhancedSimpleMetadataContributor.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/EnhancedSimpleMetadataContributor.java @@ -39,8 +39,8 @@ public class EnhancedSimpleMetadataContributor extends SimpleMetadataContributor * This method could be used to set the delimiter used during parse * If no delimiter is set, comma will be used */ - public void setDelimiter(int delimiter) { - this.delimiter = (char)delimiter; + public void setDelimiter(char delimiter) { + this.delimiter = delimiter; } /** @@ -57,8 +57,8 @@ public class EnhancedSimpleMetadataContributor extends SimpleMetadataContributor * In example, 9 for tab, 44 for comma * If no escape is set, double quote will be used */ - public void setEscape(int escape) { - this.escape = (char)escape; + public void setEscape(char escape) { + this.escape = escape; } /** @@ -91,6 +91,9 @@ public class EnhancedSimpleMetadataContributor extends SimpleMetadataContributor private String[] splitToRecord(String value) { List rows; + // For example, list of author must be: Author 1, author 2, author 3 + // if author name contains comma, is important to escape its in + // this way: Author 1, \"Author 2, something\", Author 3 try (CSVReader csvReader = new CSVReader(new StringReader(value), delimiter, escape);) { rows = csvReader.readAll(); diff --git a/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml b/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml index e2e670cc70..53f49870a7 100644 --- a/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml +++ b/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml @@ -90,7 +90,7 @@ - + diff --git a/dspace/config/spring/api/characterseparated-integration.xml b/dspace/config/spring/api/characterseparated-integration.xml index 5057acaff6..1ee62173f1 100644 --- a/dspace/config/spring/api/characterseparated-integration.xml +++ b/dspace/config/spring/api/characterseparated-integration.xml @@ -50,9 +50,6 @@ - - From 42d523639331c5b02dfd6f62572b744a1156e3c6 Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Thu, 27 Aug 2020 15:49:33 +0200 Subject: [PATCH 12/13] Add MultipleMetadataContributor to support multiple MetadataContributor for each MetadataFieldConfig --- .../MultipleMetadataContributor.java | 139 ++++++++++++++++++ .../rest/WorkspaceItemRestRepositoryIT.java | 32 ++-- .../org/dspace/app/rest/ris-test.ris | 1 + dspace/config/spring/api/ris-integration.xml | 16 +- 4 files changed, 171 insertions(+), 17 deletions(-) create mode 100644 dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/MultipleMetadataContributor.java diff --git a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/MultipleMetadataContributor.java b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/MultipleMetadataContributor.java new file mode 100644 index 0000000000..2685948fd9 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/MultipleMetadataContributor.java @@ -0,0 +1,139 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.importer.external.metadatamapping.contributor; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; + +import org.dspace.importer.external.metadatamapping.MetadataFieldConfig; +import org.dspace.importer.external.metadatamapping.MetadataFieldMapping; +import org.dspace.importer.external.metadatamapping.MetadatumDTO; + +/** + * This Contributor is helpful to avoid the limit of the Live Import Framework. + * In Live Import, one dc schema/element/qualifier could be associate with one and + * only one MetadataContributor, because the map they're saved in use dc entity as key. + * + * In fact, in this implementation we use the MetadataFieldConfig present in this MultipleMetadataContributor + * contributor, but the data (values of the dc metadatum) will be loaded using any of the contributor defined + * in the List metadatumContributors, by iterating over them. + * + * @see org.dspace.importer.external.metadatamapping.AbstractMetadataFieldMapping + * + * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it) + * + */ +public class MultipleMetadataContributor implements MetadataContributor { + + private MetadataFieldConfig field; + + private List metadatumContributors; + + /** + * Empty constructor + */ + public MultipleMetadataContributor() { + } + + /** + * @param field {@link org.dspace.importer.external.metadatamapping.MetadataFieldConfig} used in + * mapping + * @param metadatumContributors A list of MetadataContributor + */ + public MultipleMetadataContributor(MetadataFieldConfig field, List metadatumContributors) { + this.field = field; + this.metadatumContributors = (LinkedList) metadatumContributors; + } + + /** + * Set the metadatafieldMapping used in the transforming of a record to actual metadata + * + * @param metadataFieldMapping the new mapping. + */ + @Override + public void setMetadataFieldMapping(MetadataFieldMapping> metadataFieldMapping) { + for (MetadataContributor metadatumContributor : metadatumContributors) { + metadatumContributor.setMetadataFieldMapping(metadataFieldMapping); + } + } + + + /** + * a separate Metadatum object is created for each index of Metadatum returned from the calls to + * MetadatumContributor.contributeMetadata(t) for each MetadatumContributor in the metadatumContributors list. + * All of them have as dc schema/element/qualifier the values defined in MetadataFieldConfig. + * + * @param t the object we are trying to translate + * @return a collection of metadata got from each MetadataContributor + */ + @Override + public Collection contributeMetadata(T t) { + Collection values = new ArrayList<>(); + for (MetadataContributor metadatumContributor : metadatumContributors) { + Collection metadata = metadatumContributor.contributeMetadata(t); + values.addAll(metadata); + } + changeDC(values); + return values; + } + + /** + * This method does the trick of this implementation. + * It changes the DC schema/element/qualifier of the given Metadatum into + * the ones present in this contributor. + * In this way, the contributors in metadatumContributors could have any dc values, + * because this method remap them all. + * + * @param the list of metadata we want to remap + */ + private void changeDC(Collection values) { + for (MetadatumDTO dto : values) { + dto.setElement(field.getElement()); + dto.setQualifier(field.getQualifier()); + dto.setSchema(field.getSchema()); + } + } + + /** + * Return the MetadataFieldConfig used while retrieving MetadatumDTO + * + * @return MetadataFieldConfig + */ + public MetadataFieldConfig getField() { + return field; + } + + /** + * Setting the MetadataFieldConfig + * + * @param field MetadataFieldConfig used while retrieving MetadatumDTO + */ + public void setField(MetadataFieldConfig field) { + this.field = field; + } + + /** + * Return the List of MetadataContributor objects set to this class + * + * @return metadatumContributors, list of MetadataContributor + */ + public List getMetadatumContributors() { + return metadatumContributors; + } + + /** + * Set the List of MetadataContributor objects set to this class + * + * @param metadatumContributors A list of MetadatumContributor classes + */ + public void setMetadatumContributors(List metadatumContributors) { + this.metadatumContributors = metadatumContributors; + } +} diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java index e6e70ecdaa..55f590a3d4 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java @@ -990,12 +990,12 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration context.restoreAuthSystemState(); String authToken = getAuthToken(eperson.getEmail(), password); - // bulk create workspaceitems in the default collection (col1) + // create workspaceitems in the default collection (col1) AtomicReference> idRef = new AtomicReference<>(); try { getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(csvFile)) - // bulk create should return 200, 201 (created) is better for single resource + // create should return 200, 201 (created) is better for single resource .andExpect(status().isOk()) .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.title'][0].value", is("My Article"))) @@ -1030,7 +1030,7 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration } } - // bulk create workspaceitems explicitly in the col2 + // create workspaceitems explicitly in the col2 try { getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(csvFile) @@ -1107,12 +1107,12 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration String authToken = getAuthToken(eperson.getEmail(), password); AtomicReference> idRef = new AtomicReference<>(); - // bulk create workspaceitems in the default collection (col1) + // create workspaceitems in the default collection (col1) try { getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(csvFile)) - // bulk create should return 200, 201 (created) is better for single resource + // create should return 200, 201 (created) is better for single resource .andExpect(status().isOk()) .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.title'][0].value", is("My Article"))) @@ -1186,11 +1186,11 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration String authToken = getAuthToken(eperson.getEmail(), password); AtomicReference> idRef = new AtomicReference<>(); - // bulk create workspaceitems in the default collection (col1) + // create workspaceitems in the default collection (col1) try { getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(tsvFile)) - // bulk create should return 200, 201 (created) is better for single resource + // create should return 200, 201 (created) is better for single resource .andExpect(status().isOk()) .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.title'][0].value", is("My Article"))) @@ -1257,14 +1257,16 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration String authToken = getAuthToken(eperson.getEmail(), password); AtomicReference> idRef = new AtomicReference<>(); - // bulk create workspaceitems in the default collection (col1) + // create workspaceitems in the default collection (col1) try { getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(tsvFile)) - // bulk create should return 200, 201 (created) is better for single resource + // create should return 200, 201 (created) is better for single resource .andExpect(status().isOk()) .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.title'][0].value", is("Challenge–Response Identification"))) + .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.title'][1].value", + is("Challenge–Response Identification second title"))) .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone" + "['dc.contributor.author'][0].value", is("Just, Mike"))) @@ -1326,11 +1328,11 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration String authToken = getAuthToken(eperson.getEmail(), password); AtomicReference> idRef = new AtomicReference<>(); - // bulk create workspaceitems in the default collection (col1) + // create workspaceitems in the default collection (col1) try { getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(endnoteFile)) - // bulk create should return 200, 201 (created) is better for single resource + // create should return 200, 201 (created) is better for single resource .andExpect(status().isOk()) .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.title'][0].value", is("My Title"))) @@ -1405,11 +1407,11 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration String authToken = getAuthToken(eperson.getEmail(), password); AtomicReference> idRef = new AtomicReference<>(); - // bulk create workspaceitems in the default collection (col1) + // create workspaceitems in the default collection (col1) try { getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(csvFile)) - // bulk create should return 200, 201 (created) is better for single resource + // create should return 200, 201 (created) is better for single resource .andExpect(status().isOk()) .andExpect(jsonPath("$._embedded.workspaceitems[0].sections.traditionalpageone['dc.title'][0].value", is("My Article"))) @@ -1729,10 +1731,10 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration context.restoreAuthSystemState(); - // bulk create a workspaceitem + // create a workspaceitem getClient(authToken).perform(fileUpload("/api/submission/workspaceitems") .file(pdfFile)) - // bulk create should return 200, 201 (created) is better for single resource + // create should return 200, 201 (created) is better for single resource .andExpect(status().isOk()) //FIXME it will be nice to setup a mock grobid server for end to end testing // no metadata for now diff --git a/dspace-server-webapp/src/test/resources/org/dspace/app/rest/ris-test.ris b/dspace-server-webapp/src/test/resources/org/dspace/app/rest/ris-test.ris index ee3d929062..e056e6ace2 100644 --- a/dspace-server-webapp/src/test/resources/org/dspace/app/rest/ris-test.ris +++ b/dspace-server-webapp/src/test/resources/org/dspace/app/rest/ris-test.ris @@ -4,6 +4,7 @@ ED - van Tilborg, Henk C. A. PY - 2005 DA - 2005// TI - Challenge–Response Identification +T1 - Challenge–Response Identification second title BT - Encyclopedia of Cryptography and Security SP - 73 EP - 74 diff --git a/dspace/config/spring/api/ris-integration.xml b/dspace/config/spring/api/ris-integration.xml index 39b8edcf8d..3a7f0feade 100644 --- a/dspace/config/spring/api/ris-integration.xml +++ b/dspace/config/spring/api/ris-integration.xml @@ -26,10 +26,22 @@ - + + - + + + + + + + + + + + + From 46193ff16ea6c3241db3ac5f97fe7a1375d96b40 Mon Sep 17 00:00:00 2001 From: Pasquale Cavallo Date: Thu, 27 Aug 2020 16:08:21 +0200 Subject: [PATCH 13/13] Enable missing tests --- .../app/rest/WorkspaceItemRestRepositoryIT.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java index 55f590a3d4..fe784d03b0 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java @@ -1227,7 +1227,12 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration tsv.close(); } - + @Test + /** + * Test the creation of workspaceitems POSTing to the resource collection endpoint a ris file + * + * @throws Exception + */ public void createSingleWorkspaceItemFromRISWithOneEntryTest() throws Exception { context.turnOffAuthorisationSystem(); @@ -1300,6 +1305,12 @@ public class WorkspaceItemRestRepositoryIT extends AbstractControllerIntegration ris.close(); } + @Test + /** + * Test the creation of workspaceitems POSTing to the resource collection endpoint an endnote file + * + * @throws Exception + */ public void createSingleWorkspaceItemFromEndnoteWithOneEntryTest() throws Exception { context.turnOffAuthorisationSystem();