hazza/DSpace
1
0
Fork 0
mirror of https://github.com/DSpace/DSpace.git synced 2025-10-18 15:33:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

[CST-5303] refactoring of wos live import

Browse Source
This commit is contained in:
Mykhaylo
2022-04-06 12:22:27 +02:00
parent 988a8c9335
commit ee5078f93e
6 changed files with 117 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/SimpleConcatContributor.java vendored
View File

@@ -7,14 +7,20 @@
*/
package org.dspace.importer.external.metadatamapping.contributor;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.dspace.importer.external.metadatamapping.MetadatumDTO;
import org.jdom2.Element;
import org.jdom2.Namespace;
import org.jdom2.filter.Filters;
import org.jdom2.xpath.XPathExpression;
import org.jdom2.xpath.XPathFactory;
/**
* This contributor is able to concat multi value.
@@ -25,16 +31,26 @@ import org.jdom2.Namespace;
*/
public class SimpleConcatContributor extends SimpleXpathMetadatumContributor {
private final static Logger log = LogManager.getLogger();
@Override
public Collection<MetadatumDTO> contributeMetadata(Element element) {
public Collection<MetadatumDTO> contributeMetadata(Element t) {
List<MetadatumDTO> values = new LinkedList<>();
StringBuilder text = new StringBuilder();
List<Namespace> namespaces = new ArrayList<Namespace>();
for (String ns : prefixToNamespaceMapping.keySet()) {
List<Element> nodes = element.getChildren(query, Namespace.getNamespace(ns));
for (Element el : nodes) {
if (StringUtils.isNotBlank(el.getValue())) {
namespaces.add(Namespace.getNamespace(prefixToNamespaceMapping.get(ns), ns));
}
XPathExpression<Object> xpath = XPathFactory.instance().compile(query, Filters.fpassthrough(), null,namespaces);
List<Object> nodes = xpath.evaluate(t);
for (Object el : nodes) {
if (el instanceof Element) {
Element element = (Element) el;
if (StringUtils.isNotBlank(element.getText())) {
text.append(element.getText());
}
} else {
log.warn("node of type: " + el.getClass());
}
}
if (StringUtils.isNotBlank(text.toString())) {

24
dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/SimpleMultiplePathContributor.java vendored
View File

@@ -7,13 +7,19 @@
*/
package org.dspace.importer.external.metadatamapping.contributor;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.dspace.importer.external.metadatamapping.MetadatumDTO;
import org.jdom2.Element;
import org.jdom2.Namespace;
import org.jdom2.filter.Filters;
import org.jdom2.xpath.XPathExpression;
import org.jdom2.xpath.XPathFactory;
/**
* This contributor can perform research on multi-paths
@@ -22,6 +28,8 @@ import org.jdom2.Namespace;
*/
public class SimpleMultiplePathContributor extends SimpleXpathMetadatumContributor {
private final static Logger log = LogManager.getLogger();
private List<String> paths;
public SimpleMultiplePathContributor() {}
@@ -31,13 +39,21 @@ public class SimpleMultiplePathContributor extends SimpleXpathMetadatumContribut
}
@Override
public Collection<MetadatumDTO> contributeMetadata(Element element) {
public Collection<MetadatumDTO> contributeMetadata(Element t) {
List<MetadatumDTO> values = new LinkedList<>();
for (String path : this.paths) {
List<Namespace> namespaces = new ArrayList<Namespace>();
for (String ns : prefixToNamespaceMapping.keySet()) {
List<Element> nodes = element.getChildren(path, Namespace.getNamespace(ns));
for (Element el : nodes) {
values.add(metadataFieldMapping.toDCValue(field, el.getValue()));
namespaces.add(Namespace.getNamespace(prefixToNamespaceMapping.get(ns), ns));
}
XPathExpression<Object> xpath = XPathFactory.instance().compile(path, Filters.fpassthrough(), null,
namespaces);
List<Object> nodes = xpath.evaluate(t);
for (Object el : nodes) {
if (el instanceof Element) {
values.add(metadataFieldMapping.toDCValue(field, ((Element) el).getText()));
} else {
log.warn("node of type: " + el.getClass());
}
}
}

28
dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/SimpleXpathMetadatumAndAttributeContributor.java vendored
View File

@@ -7,15 +7,21 @@
*/
package org.dspace.importer.external.metadatamapping.contributor;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.dspace.core.Constants;
import org.dspace.importer.external.metadatamapping.MetadatumDTO;
import org.jdom2.Element;
import org.jdom2.Namespace;
import org.jdom2.filter.Filters;
import org.jdom2.xpath.XPathExpression;
import org.jdom2.xpath.XPathFactory;
/**
* This contributor checks for each node returned for the supplied path
@@ -26,20 +32,32 @@ import org.jdom2.Namespace;
*/
public class SimpleXpathMetadatumAndAttributeContributor extends SimpleXpathMetadatumContributor {
private final static Logger log = LogManager.getLogger();
private String attribute;
@Override
public Collection<MetadatumDTO> contributeMetadata(Element element) {
public Collection<MetadatumDTO> contributeMetadata(Element t) {
List<MetadatumDTO> values = new LinkedList<>();
List<Namespace> namespaces = new ArrayList<Namespace>();
for (String ns : prefixToNamespaceMapping.keySet()) {
List<Element> nodes = element.getChildren(query, Namespace.getNamespace(ns));
for (Element el : nodes) {
String attributeValue = el.getAttributeValue(this.attribute);
namespaces.add(Namespace.getNamespace(prefixToNamespaceMapping.get(ns), ns));
}
XPathExpression<Object> xpath = XPathFactory.instance().compile(query, Filters.fpassthrough(), null,
namespaces);
List<Object> nodes = xpath.evaluate(t);
for (Object el : nodes) {
if (el instanceof Element) {
Element element = (Element) el;
String attributeValue = element.getAttributeValue(this.attribute);
if (StringUtils.isNotBlank(attributeValue)) {
values.add(metadataFieldMapping.toDCValue(this.field, attributeValue));
} else {
values.add(metadataFieldMapping.toDCValue(this.field, Constants.PLACEHOLDER_PARENT_METADATA_VALUE));
values.add(metadataFieldMapping.toDCValue(this.field,
Constants.PLACEHOLDER_PARENT_METADATA_VALUE));
}
} else {
log.warn("node of type: " + el.getClass());
}
}
return values;

43
dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/WosAttribute2ValueContributor.java vendored
View File

@@ -7,6 +7,7 @@
*/
package org.dspace.importer.external.metadatamapping.contributor;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
@@ -14,14 +15,16 @@ import java.util.Map;
import javax.annotation.Resource;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.dspace.importer.external.metadatamapping.MetadataFieldConfig;
import org.dspace.importer.external.metadatamapping.MetadataFieldMapping;
import org.dspace.importer.external.metadatamapping.MetadatumDTO;
import org.jaxen.JaxenException;
import org.jdom2.Element;
import org.jdom2.Namespace;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.jdom2.filter.Filters;
import org.jdom2.xpath.XPathExpression;
import org.jdom2.xpath.XPathFactory;
/**
* This contributor checks for each node returned for the given path if the node contains "this.attribute"
@@ -33,7 +36,7 @@ import org.slf4j.LoggerFactory;
*/
public class WosAttribute2ValueContributor implements MetadataContributor<Element> {
private static final Logger log = LoggerFactory.getLogger(WosAttribute2ValueContributor.class);
private final static Logger log = LogManager.getLogger();
private String query;
@@ -60,24 +63,28 @@ public class WosAttribute2ValueContributor implements MetadataContributor<Elemen
}
@Override
public Collection<MetadatumDTO> contributeMetadata(Element element) {
public Collection<MetadatumDTO> contributeMetadata(Element t) {
List<MetadatumDTO> values = new LinkedList<>();
try {
for (String ns : prefixToNamespaceMapping.keySet()) {
List<Element> nodes = element.getChildren(query, Namespace.getNamespace(ns));
for (Element el : nodes) {
String attributeValue = el.getAttributeValue(this.attribute);
setField(attributeValue, element, values);
}
}
return values;
} catch (JaxenException e) {
log.warn(query, e);
throw new RuntimeException(e);
List<Namespace> namespaces = new ArrayList<Namespace>();
for (String ns : prefixToNamespaceMapping.keySet()) {
namespaces.add(Namespace.getNamespace(prefixToNamespaceMapping.get(ns), ns));
}
XPathExpression<Object> xpath = XPathFactory.instance().compile(query, Filters.fpassthrough(), null,
namespaces);
List<Object> nodes = xpath.evaluate(t);
for (Object el : nodes) {
if (el instanceof Element) {
Element element = (Element) el;
String attributeValue = element.getAttributeValue(this.attribute);
setField(attributeValue, element, values);
} else {
log.warn("node of type: " + el.getClass());
}
}
return values;
}
private void setField(String attributeValue, Element el, List<MetadatumDTO> values) throws JaxenException {
private void setField(String attributeValue, Element el, List<MetadatumDTO> values) {
for (String id : attributeValue2metadata.keySet()) {
if (StringUtils.equals(id, attributeValue)) {
if (this.firstChild) {

48
dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/WosIdentifierRidContributor.java vendored
View File

@@ -7,19 +7,22 @@
*/
package org.dspace.importer.external.metadatamapping.contributor;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
import java.util.Objects;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.dspace.core.Constants;
import org.dspace.importer.external.metadatamapping.MetadatumDTO;
import org.jaxen.JaxenException;
import org.jdom2.Element;
import org.jdom2.Namespace;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.jdom2.filter.Filters;
import org.jdom2.xpath.XPathExpression;
import org.jdom2.xpath.XPathFactory;
/**
*
@@ -27,30 +30,33 @@ import org.slf4j.LoggerFactory;
*/
public class WosIdentifierRidContributor extends SimpleXpathMetadatumContributor {
private static final Logger log = LoggerFactory.getLogger(WosIdentifierRidContributor.class);
private final static Logger log = LogManager.getLogger();
@Override
public Collection<MetadatumDTO> contributeMetadata(Element element) {
public Collection<MetadatumDTO> contributeMetadata(Element t) {
List<MetadatumDTO> values = new LinkedList<>();
try {
for (String ns : prefixToNamespaceMapping.keySet()) {
List<Element> nodes = element.getChildren(query, Namespace.getNamespace(ns));
for (Element el : nodes) {
// Element element2 = el.getFirstChildWithName("name");
if (Objects.nonNull(element)) {
String type = element.getAttributeValue("role");
setIdentyfire(type, element, values);
}
}
}
return values;
} catch (JaxenException e) {
log.error(query, e);
throw new RuntimeException(e);
List<Namespace> namespaces = new ArrayList<Namespace>();
for (String ns : prefixToNamespaceMapping.keySet()) {
namespaces.add(Namespace.getNamespace(prefixToNamespaceMapping.get(ns), ns));
}
XPathExpression<Object> xpath = XPathFactory.instance().compile(query, Filters.fpassthrough(), null,
namespaces);
List<Object> nodes = xpath.evaluate(t);
for (Object el : nodes) {
if (el instanceof Element) {
Element element = ((Element) el).getChild("name");
if (Objects.nonNull(element)) {
String type = element.getAttributeValue("role");
setIdentyfire(type, element, values);
}
} else {
log.warn("node of type: " + el.getClass());
}
}
return values;
}
private void setIdentyfire(String type, Element el, List<MetadatumDTO> values) throws JaxenException {
private void setIdentyfire(String type, Element el, List<MetadatumDTO> values) {
if (StringUtils.equals("researcher_id", type)) {
String value = el.getAttributeValue("r_id");
if (StringUtils.isNotBlank(value)) {

4
dspace-server-webapp/src/test/java/org/dspace/app/rest/AbstractLiveImportIntegrationTest.java
View File

@@ -32,8 +32,8 @@ public class AbstractLiveImportIntegrationTest extends AbstractControllerIntegra
ImportRecord secondImported = recordsImported.iterator().next();
ImportRecord first2match = recordsImported.iterator().next();
ImportRecord second2match = recordsImported.iterator().next();
boolean checkFirstRecord = firstImported.getValueList().containsAll(first2match.getValueList());
boolean checkSecondRecord = secondImported.getValueList().containsAll(second2match.getValueList());
boolean checkFirstRecord = first2match.getValueList().containsAll(firstImported.getValueList());
boolean checkSecondRecord = second2match.getValueList().containsAll(secondImported.getValueList());
return checkFirstRecord && checkSecondRecord;
}