From 14a91bfb3820c0dc2ae64c3d4f91c41cafe0e3fe Mon Sep 17 00:00:00 2001 From: Ben Bosman Date: Tue, 5 May 2020 09:42:59 +0200 Subject: [PATCH 1/4] support empty values in the CSV deleting the data --- .../main/java/org/dspace/app/bulkedit/MetadataImport.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImport.java b/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImport.java index e8fff71cf4..785452392e 100644 --- a/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImport.java +++ b/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImport.java @@ -1464,16 +1464,16 @@ public class MetadataImport { //Populate the EntityRelationMap populateEntityRelationMap(uuid, key, originId.toString()); } + } else { + newLine.add(key, null); } } else { - if (line.get(key).size() > 1) { + if (line.get(key).size() > 0) { for (String value : line.get(key)) { newLine.add(key, value); } } else { - if (line.get(key).size() > 0) { - newLine.add(key, line.get(key).get(0)); - } + newLine.add(key, null); } } } From 6cdab6effda4e8eb8d1519bd72f783ca161ef19d Mon Sep 17 00:00:00 2001 From: Raf Ponsaerts Date: Mon, 26 Oct 2020 14:38:36 +0100 Subject: [PATCH 2/4] Rewrote MetadataImportTest and added a test for empty value support --- .../app/bulkedit/MetadataImportTest.java | 105 ++++++++++++++---- 1 file changed, 85 insertions(+), 20 deletions(-) diff --git a/dspace-api/src/test/java/org/dspace/app/bulkedit/MetadataImportTest.java b/dspace-api/src/test/java/org/dspace/app/bulkedit/MetadataImportTest.java index 4a0043586b..a91c8b45d8 100644 --- a/dspace-api/src/test/java/org/dspace/app/bulkedit/MetadataImportTest.java +++ b/dspace-api/src/test/java/org/dspace/app/bulkedit/MetadataImportTest.java @@ -10,66 +10,98 @@ package org.dspace.app.bulkedit; import static junit.framework.TestCase.assertEquals; import static junit.framework.TestCase.assertTrue; +import java.io.BufferedWriter; import java.io.File; +import java.io.FileOutputStream; +import java.io.OutputStreamWriter; +import java.sql.SQLException; +import java.util.List; import org.apache.commons.cli.ParseException; +import org.apache.commons.collections4.IteratorUtils; import org.apache.commons.lang3.StringUtils; -import org.dspace.AbstractIntegrationTest; +import org.dspace.AbstractIntegrationTestWithDatabase; import org.dspace.app.launcher.ScriptLauncher; import org.dspace.app.scripts.handler.impl.TestDSpaceRunnableHandler; +import org.dspace.builder.CollectionBuilder; +import org.dspace.builder.CommunityBuilder; +import org.dspace.builder.ItemBuilder; import org.dspace.content.Collection; import org.dspace.content.Community; import org.dspace.content.Item; import org.dspace.content.factory.ContentServiceFactory; -import org.dspace.content.service.CollectionService; -import org.dspace.content.service.CommunityService; import org.dspace.content.service.ItemService; +import org.dspace.eperson.factory.EPersonServiceFactory; +import org.dspace.eperson.service.EPersonService; import org.dspace.scripts.DSpaceRunnable; import org.dspace.scripts.configuration.ScriptConfiguration; import org.dspace.scripts.factory.ScriptServiceFactory; import org.dspace.scripts.service.ScriptService; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -public class MetadataImportTest extends AbstractIntegrationTest { +public class MetadataImportTest extends AbstractIntegrationTestWithDatabase { private final ItemService itemService = ContentServiceFactory.getInstance().getItemService(); - private final CollectionService collectionService - = ContentServiceFactory.getInstance().getCollectionService(); - private final CommunityService communityService - = ContentServiceFactory.getInstance().getCommunityService(); + private final EPersonService ePersonService = EPersonServiceFactory.getInstance().getEPersonService(); + private Collection collection; @Rule public ExpectedException thrown = ExpectedException.none(); + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + context.turnOffAuthorisationSystem(); + Community community = CommunityBuilder.createCommunity(context).build(); + this.collection = CollectionBuilder.createCollection(context, community).build(); + context.restoreAuthSystemState(); + } + @Test public void metadataImportTest() throws Exception { - context.turnOffAuthorisationSystem(); - Community community = communityService.create(null, context); - Collection collection = collectionService.create(context, community); - context.restoreAuthSystemState(); + String[] csv = {"id,collection,dc.title,dc.contributor.author", + "+," + collection.getHandle() + ",\"Test Import 1\"," + "\"Donald, SmithImported\""}; + performImportScript(csv); + Item importedItem = findItemByName("Test Import 1"); - String fileLocation = new File(testProps.get("test.importcsv").toString()).getAbsolutePath(); - String[] args = new String[] {"metadata-import", "-f", fileLocation, "-e", eperson.getEmail(), "-s"}; - TestDSpaceRunnableHandler testDSpaceRunnableHandler = new TestDSpaceRunnableHandler(); - - ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), testDSpaceRunnableHandler, kernelImpl); - Item importedItem = itemService.findAll(context).next(); assertTrue( StringUtils.equals( itemService.getMetadata(importedItem, "dc", "contributor", "author", Item.ANY).get(0).getValue(), "Donald, SmithImported")); + + eperson = ePersonService.findByEmail(context, eperson.getEmail()); assertEquals(importedItem.getSubmitter(), eperson); context.turnOffAuthorisationSystem(); itemService.delete(context, itemService.find(context, importedItem.getID())); - collectionService.delete(context, collectionService.find(context, collection.getID())); - communityService.delete(context, communityService.find(context, community.getID())); context.restoreAuthSystemState(); } + @Test + public void metadataImportRemovingValueTestTest() throws Exception { + + context.turnOffAuthorisationSystem(); + Item item = ItemBuilder.createItem(context, collection).withAuthor("TestAuthorToRemove").withTitle("title") + .build(); + context.restoreAuthSystemState(); + + assertTrue( + StringUtils.equals( + itemService.getMetadata(item, "dc", "contributor", "author", Item.ANY).get(0).getValue(), + "TestAuthorToRemove")); + + String[] csv = {"id,collection,dc.title,dc.contributor.author[*]", + item.getID().toString() + "," + collection.getHandle() + "," + item.getName() + ","}; + performImportScript(csv); + item = findItemByName("title"); + assertEquals(itemService.getMetadata(item, "dc", "contributor", "author", Item.ANY).size(), 0); + } + @Test(expected = ParseException.class) public void metadataImportWithoutEPersonParameterTest() throws IllegalAccessException, InstantiationException, ParseException { @@ -89,4 +121,37 @@ public class MetadataImportTest extends AbstractIntegrationTest { script.run(); } } + + private Item findItemByName(String name) throws SQLException { + Item importedItem = null; + List allItems = IteratorUtils.toList(itemService.findAll(context)); + for (Item item : allItems) { + if (item.getName().equals(name)) { + importedItem = item; + } + } + return importedItem; + } + + /** + * Import mocked CSVs to test item creation behavior, deleting temporary file afterward. + */ + public void performImportScript(String[] csv) throws Exception { + File csvFile = File.createTempFile("dspace-test-import", "csv"); + BufferedWriter out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(csvFile), "UTF-8")); + for (String csvLine : csv) { + out.write(csvLine + "\n"); + } + out.flush(); + out.close(); + String fileLocation = csvFile.getAbsolutePath(); + try { + String[] args = new String[] {"metadata-import", "-f", fileLocation, "-e", eperson.getEmail(), "-s"}; + TestDSpaceRunnableHandler testDSpaceRunnableHandler = new TestDSpaceRunnableHandler(); + ScriptLauncher + .handleScript(args, ScriptLauncher.getConfig(kernelImpl), testDSpaceRunnableHandler, kernelImpl); + } finally { + csvFile.delete(); + } + } } From ea7d5d5b79f1621e2c3271ffe905af1ea34af7ff Mon Sep 17 00:00:00 2001 From: Yana De Pauw Date: Wed, 28 Oct 2020 12:29:31 +0100 Subject: [PATCH 3/4] 74186: CSV import error with virtual metadata bugfix --- .../dspace/app/bulkedit/MetadataImport.java | 7 +++++-- .../app/csv/CSVMetadataImportReferenceIT.java | 20 +++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImport.java b/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImport.java index 67086c1536..cb4ef6dd1e 100644 --- a/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImport.java +++ b/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataImport.java @@ -113,14 +113,14 @@ public class MetadataImport extends DSpaceRunnable entityTypeMap = new HashMap<>(); + protected HashMap entityTypeMap = new HashMap<>(); /** * Map of UUIDs to their relations that are referenced within any import with their referers. * * @see #populateEntityRelationMap(String, String, String) */ - protected static HashMap>> entityRelationMap = new HashMap<>(); + protected HashMap>> entityRelationMap = new HashMap<>(); /** @@ -1517,6 +1517,9 @@ public class MetadataImport extends DSpaceRunnable Date: Wed, 18 Nov 2020 13:38:47 +0100 Subject: [PATCH 4/4] Re-added the metadataImportRemovingValueTest in MetadataImportIT --- .../dspace/app/bulkedit/MetadataImportIT.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/dspace-api/src/test/java/org/dspace/app/bulkedit/MetadataImportIT.java b/dspace-api/src/test/java/org/dspace/app/bulkedit/MetadataImportIT.java index 3dd229f066..da89bc9950 100644 --- a/dspace-api/src/test/java/org/dspace/app/bulkedit/MetadataImportIT.java +++ b/dspace-api/src/test/java/org/dspace/app/bulkedit/MetadataImportIT.java @@ -152,6 +152,26 @@ public class MetadataImportIT extends AbstractIntegrationTestWithDatabase { } + @Test + public void metadataImportRemovingValueTest() throws Exception { + + context.turnOffAuthorisationSystem(); + Item item = ItemBuilder.createItem(context, collection).withAuthor("TestAuthorToRemove").withTitle("title") + .build(); + context.restoreAuthSystemState(); + + assertTrue( + StringUtils.equals( + itemService.getMetadata(item, "dc", "contributor", "author", Item.ANY).get(0).getValue(), + "TestAuthorToRemove")); + + String[] csv = {"id,collection,dc.title,dc.contributor.author[*]", + item.getID().toString() + "," + collection.getHandle() + "," + item.getName() + ","}; + performImportScript(csv); + item = findItemByName("title"); + assertEquals(itemService.getMetadata(item, "dc", "contributor", "author", Item.ANY).size(), 0); + } + private Item findItemByName(String name) throws SQLException { Item importedItem = null; List allItems = IteratorUtils.toList(itemService.findAll(context));