diff --git a/dspace-api/pom.xml b/dspace-api/pom.xml index 4920ad3808..0e9b2e78ed 100644 --- a/dspace-api/pom.xml +++ b/dspace-api/pom.xml @@ -670,6 +670,18 @@ 1 jar + + + org.apache.ws.commons.axiom + axiom-impl + 1.2.14 + + + + org.apache.ws.commons.axiom + axiom-api + 1.2.14 + com.amazonaws diff --git a/dspace-api/src/main/java/org/dspace/importer/external/MetadataSourceException.java b/dspace-api/src/main/java/org/dspace/importer/external/MetadataSourceException.java new file mode 100644 index 0000000000..13739852ac --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/MetadataSourceException.java @@ -0,0 +1,31 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ + +package org.dspace.importer.external; + +/** Represents a problem with the input source: e.g. cannot connect to the source. + * Created by Roeland Dillen (roeland at atmire dot com) + * Date: 19/09/12 + * Time: 13:17 + */ +public class MetadataSourceException extends Exception { + public MetadataSourceException() { + } + + public MetadataSourceException(String s) { + super(s); + } + + public MetadataSourceException(String s, Throwable throwable) { + super(s, throwable); + } + + public MetadataSourceException(Throwable throwable) { + super(throwable); + } +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/Query.java b/dspace-api/src/main/java/org/dspace/importer/external/Query.java new file mode 100644 index 0000000000..cf82afaaef --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/Query.java @@ -0,0 +1,55 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ + +package org.dspace.importer.external; + +import org.apache.commons.collections.map.MultiValueMap; + +import java.util.Collection; + +/** + * Created by Roeland Dillen (roeland at atmire dot com) + * Date: 27/09/12 + * Time: 15:26 + */ +public class Query { + private MultiValueMap parameters = new MultiValueMap(); + + public MultiValueMap getParameters() { + return parameters; + } + + public void addParameter(String key,Object value){ + parameters.put(key,value); + } + + protected void addSingletonParameter(String key,Object value){ + parameters.remove(key); + parameters.put(key,value); + } + + public T getParameterAsClass(String key, Class clazz){ + Collection c=parameters.getCollection(key); + if(c==null||c.isEmpty()) return null; + else { + Object o=c.iterator().next(); + if(clazz.isAssignableFrom(o.getClass())) + return (T) o ; + else return null; + } + + } + + public Collection getParameter(String key){ + return parameters.getCollection(key); + } + + public void setParameters(MultiValueMap parameters) { + this.parameters = parameters; + } +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/README.md b/dspace-api/src/main/java/org/dspace/importer/external/README.md new file mode 100644 index 0000000000..e55aed8299 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/README.md @@ -0,0 +1,147 @@ +- [Introduction](#Introduction) + - [Features](#Features) + - [Abstraction of input format](#Abstraction-input-format) + - [Transformation to DSpace item](#transformation) + - [Relation with BTE](#bte) +- [Implementation of an import source](#Example-implementation) + - [Inherited methods](#Inherited-methods) + - [Metadata mapping](#Mapping) + + +# Introduction # + +This documentation explains the features and the usage of the importer framework. + +## Features ## + +- lookup publications from remote sources +- Support for multiple implementations + +## Abstraction of input format ## + +The importer framework does not enforce a specific input format. Each importer implementation defines which input format it expects from a remote source. +The import framework uses generics to achieve this. Each importer implementation will have a type set of the record type it receives from the remote source's response. +This type set will also be used by the framework to use the correct MetadataFieldMapping for a certain implementation. Read [Implementation of an import source](#Example-implementation) for more information. + +## Transformation to DSpace item ## + +The framework produces an 'ImportRecord' that is completely decoupled from DSPace. It contains a set of metadata DTO's that contain the notion of schema,element and qualifier. The specific implementation is responsible for populating this set. It is then very simple to create a DSPace item from this list. + +## Relation with BTE ## + +While there is some overlap between this framework and BTE, this framework supports some features that are hard to implement using the BTE. It has explicit support to deal with network failure and throttling imposed by the data source. It also has explicit support for distinguishing between network caused errors and invalid requests to the source. +Furthermore the framework doesn't impose any restrictions on the format in which the data is retrieved. It uses java generics to support different source record types. A reference implementation of using XML records is provided for which a set of metadata can be generated from any xpath expression (or composite of xpath expressions). +Unless 'advanced' processing is necessary (e.g. lookup of authors in an LDAP directory) this metadata mapping can be simply configured using spring. No code changes necessary. A mixture of advanced and simple (xpath) mapping is also possible. + +This design is also in line with the roadmap to create a Modular Framework as detailed in [https://wiki.duraspace.org/display/DSPACE/Design+-+Module+Framework+and+Registry](https://wiki.duraspace.org/display/DSPACE/Design+-+Module+Framework+and+Registry) +This modular design also allows it to be completely independent of the user interface layer, be it JSPUI, XMLUI, command line or the result of the new UI projects: [https://wiki.duraspace.org/display/DSPACE/Design+-+Single+UI+Project](https://wiki.duraspace.org/display/DSPACE/Design+-+Single+UI+Project) + +# Implementation of an import source # + +Each importer implementation must at least implement interface *org.dspace.importer.external.service.other.Imports* and implement the inherited methods. + +One can also choose to implement class *org.dspace.importer.external.service.other.Source* next to the Imports interface. This class contains functionality to handle request timeouts and to retry requests. + +A third option is to implement class *org.dspace.importer.external.service.AbstractImportSourceService*. This class already implements both the Imports interface and Source class. AbstractImportSourceService has a generic type set 'RecordType'. In the importer implementation this type set should be the class of the records received from the remote source's response (e.g. when using axiom to get the records from the remote source's XML response, the importer implementation's type set is *org.apache.axiom.om.OMElement*). + +Implementing the AbstractImportSourceService allows the importer implementation to use the framework's build-in support to transform a record received from the remote source to an object of class *org.dspace.importer.external.datamodel.ImportRecord* containing DSpace metadata fields, as explained here: [Metadata mapping](#Mapping). + +## Inherited methods ## + +Method getImportSource() should return a unique identifier. Importer implementations should not be called directly, but class *org.dspace.importer.external.service.ImportService* should be called instead. This class contains the same methods as the importer implementatons, but with an extra parameter 'url'. This url parameter should contain the same identifier that is returned by the getImportSource() method of the importer implementation you want to use. + +The other inherited methods are used to query the remote source. + +## Metadata mapping ## + +When using an implementation of AbstractImportSourceService, a mapping of remote record fields to DSpace metadata fields can be created. + +first create an implementation of class AbstractMetadataFieldMapping with the same type set used for the importer implementation. + +Then create a spring configuration file in [dspace.dir]/config/spring/api. + +Each DSpace metadata field that will be used for the mapping must first be configured as a spring bean of class *org.dspace.importer.external.metadatamapping.MetadataFieldConfig*. + +```xml + + + +``` + +Now this metadata field can be used to create a mapping. To add a mapping for the "dc.title" field declared above, a new spring bean configuration of a class class *org.dspace.importer.external.metadatamapping.contributor.MetadataContributor* needs to be added. This interface contains a type argument. +The type needs to match the type used in the implementation of AbstractImportSourceService. The responsibility of each MetadataContributor implementation is to generate a set of metadata from the retrieved document. How it does that is completely opaque to the AbstractImportSourceService but it is assumed that only one entity (i.e. item) is fed to the metadatum contributor. + + +For example ```java SimpleXpathMetadatumContributor implements MetadataContributor``` can parse a fragment of xml and generate one or more metadata values. + + +This bean expects 2 property values: + +- field: A reference to the configured spring bean of the DSpace metadata field. e.g. the "dc.title" bean declared above. +- query: The xpath expression used to select the record value returned by the remote source. + +```xml + + + + +``` + +Multiple record fields can also be combined into one value. To implement a combined mapping first create a *SimpleXpathMetadatumContributor* as explained above for each part of the field. + +```xml + + + + + + + + +``` + +Note that namespace prefixes used in the xpath queries are configured in bean "FullprefixMapping" in the same spring file. + +```xml + + Defines the namespace mappin for the SimpleXpathMetadatum contributors + + + +``` + +Then create a new list in the spring configuration containing references to all *SimpleXpathMetadatumContributor* beans that need to be combined. + +```xml + + + + {{/code}} +``` + +Finally create a spring bean configuration of class *org.dspace.importer.external.metadatamapping.contributor.CombinedMetadatumContributor*. This bean expects 3 values: + +- field: A reference to the configured spring bean of the DSpace metadata field. e.g. the "dc.title" bean declared above. +- metadatumContributors: A reference to the list containing all the single record field mappings that need to be combined. +- separator: These characters will be added between each record field value when they are combined into one field. + +```xml + + + + + +``` + +Each contributor must also be added to the "MetadataFieldMap" used by the *MetadataFieldMapping* implementation. Each entry of this map maps a metadata field bean to a contributor. For the contributors created above this results in the following configuration: + +```xml + + + + +``` + +Note that the single field mappings used for the combined author mapping are not added to this list. + diff --git a/dspace-api/src/main/java/org/dspace/importer/external/SourceExceptionHandler.java b/dspace-api/src/main/java/org/dspace/importer/external/SourceExceptionHandler.java new file mode 100644 index 0000000000..84e7ef67b3 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/SourceExceptionHandler.java @@ -0,0 +1,21 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ + +package org.dspace.importer.external; + +import org.dspace.importer.external.service.other.MetadataSource; + +/** + * Created by: Antoine Snyers (antoine at atmire dot com) + * Date: 27 Oct 2014 + */ +public abstract interface SourceExceptionHandler { + + public abstract void handle(T source); + +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/datamodel/ImportRecord.java b/dspace-api/src/main/java/org/dspace/importer/external/datamodel/ImportRecord.java new file mode 100644 index 0000000000..ee253afba4 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/datamodel/ImportRecord.java @@ -0,0 +1,75 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.importer.external.datamodel; + +import org.dspace.importer.external.metadatamapping.MetadatumDTO; + +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; + +/** + * Created by Roeland Dillen (roeland at atmire dot com) + * Date: 17/09/12 + * Time: 14:03 + */ +public class ImportRecord { + private List valueList = null; + + public List getValueList() { + return Collections.unmodifiableList(valueList); + } + + public ImportRecord(List valueList) { + //don't want to alter the original list. Also now I can control the type of list + this.valueList = new LinkedList(valueList); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + sb.append("Record"); + sb.append("{valueList="); + for(MetadatumDTO val:valueList){ + sb.append("{"); + sb.append(val.getSchema()); + sb.append("; "); + sb.append(val.getElement()); + sb.append("; "); + + sb.append(val.getQualifier()); + sb.append("; "); + + sb.append(val.getValue()); + sb.append("; "); + sb.append("}\n"); + + } + sb.append("}\n"); + return sb.toString(); + } + + public Collection getValue(String schema, String element, String qualifier){ + List values=new LinkedList(); + for(MetadatumDTO value:valueList){ + if(value.getSchema().equals(schema)&&value.getElement().equals(element)){ + if(qualifier==null&&value.getQualifier()==null){ + values.add(value); + } else if (value.getQualifier()!=null&&value.getQualifier().equals(qualifier)) { + values.add(value); + } + } + } + return values; + } + + public void addValue(MetadatumDTO value){ + this.valueList.add(value); + } +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/AbstractMetadataFieldMapping.java b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/AbstractMetadataFieldMapping.java new file mode 100644 index 0000000000..1c77628bb0 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/AbstractMetadataFieldMapping.java @@ -0,0 +1,127 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.importer.external.metadatamapping; + + +import org.apache.log4j.Logger; +import org.dspace.importer.external.metadatamapping.contributor.MetadataContributor; +import org.dspace.importer.external.metadatamapping.service.MetadataProcessorService; + +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * Created by Roeland Dillen (roeland at atmire dot com) + * Date: 19/09/12 + * Time: 10:09 + */ +public abstract class AbstractMetadataFieldMapping implements MetadataFieldMapping> { + + private Map> metadataFieldMap; + + /** + * log4j logger + */ + private static Logger log = Logger.getLogger(AbstractMetadataFieldMapping.class); + + private Map metadataProcessorMap; + + public void setMetadataProcessorMap(Map metadataProcessorMap) + { + this.metadataProcessorMap = metadataProcessorMap; + } + + public MetadataProcessorService getMetadataProcessor(MetadataFieldConfig metadataField) + { + if(metadataProcessorMap != null) + { + return metadataProcessorMap.get(metadataField); + }else{ + return null; + } + } + + public MetadatumDTO toDCValue(MetadataFieldConfig field, String value) { + MetadatumDTO dcValue = new MetadatumDTO(); + + + + if (field == null) return null; + MetadataProcessorService metadataProcessor = getMetadataProcessor(field); + if(metadataProcessor != null) + { + value = metadataProcessor.processMetadataValue(value); + } + dcValue.setValue(value); + dcValue.setElement(field.getElement()); + dcValue.setQualifier(field.getQualifier()); + dcValue.setSchema(field.getSchema()); + return dcValue; + } + + private boolean reverseDifferent = false; + + private String AND = "AND"; + private String OR = "OR"; + private String NOT = "NOT"; + + public String getAND() { + return AND; + } + + public void setAND(String AND) { + this.AND = AND; + } + + public String getOR() { + return OR; + } + + public void setOR(String OR) { + this.OR = OR; + } + + public String getNOT() { + return NOT; + } + + public void setNOT(String NOT) { + this.NOT = NOT; + } + + public Map> getMetadataFieldMap() { + return metadataFieldMap; + } + + public void setMetadataFieldMap(Map> metadataFieldMap) { + this.metadataFieldMap = metadataFieldMap; + for(MetadataContributor mc:metadataFieldMap.values()){ + mc.setMetadataFieldMapping(this); + } + + } + + @Override + public Collection resultToDCValueMapping(RecordType record) { + List values=new LinkedList(); + + + for(MetadataContributor query:getMetadataFieldMap().values()){ + try { + values.addAll(query.contributeMetadata(record)); + } catch (Exception e) { + log.error("Error", e); + } + + } + return values; + + } +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/MetadataFieldConfig.java b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/MetadataFieldConfig.java new file mode 100644 index 0000000000..357c14a71d --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/MetadataFieldConfig.java @@ -0,0 +1,117 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.importer.external.metadatamapping; + +/** + * Created by Roeland Dillen (roeland at atmire dot com) + * Date: 19/09/12 + * Time: 10:11 + */ +public class MetadataFieldConfig { + private String schema; + private String element; + private String qualifier; + + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + MetadataFieldConfig that = (MetadataFieldConfig) o; + + if (!element.equals(that.element)) return false; + if (qualifier != null ? !qualifier.equals(that.qualifier) : that.qualifier != null) return false; + if (!schema.equals(that.schema)) return false; + + return true; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + sb.append("MetadataField"); + sb.append("{schema='").append(schema).append('\''); + sb.append(", element='").append(element).append('\''); + sb.append(", qualifier='").append(qualifier).append('\''); + sb.append('}'); + return sb.toString(); + } + + @Override + public int hashCode() { + int result = schema.hashCode(); + result = 31 * result + element.hashCode(); + result = 31 * result + (qualifier != null ? qualifier.hashCode() : 0); + return result; + } + + public String getSchema() { + + return schema; + } + + public MetadataFieldConfig(MetadatumDTO value) { + this.schema = value.getSchema(); + this.element = value.getElement(); + this.qualifier = value.getQualifier(); + } + + public MetadataFieldConfig() { + } + + public MetadataFieldConfig(String schema, String element, String qualifier) { + this.schema = schema; + this.element = element; + this.qualifier = qualifier; + } + + public MetadataFieldConfig(String full) { + String elements[]=full.split("\\."); + if(elements.length==2){ + this.schema = elements[0]; + this.element =elements[1]; + } else if(elements.length==3){ + this.schema = elements[0]; + this.element =elements[1]; + this.qualifier = elements[2]; + } + + } + + public MetadataFieldConfig(String schema, String element) { + this.schema = schema; + this.element = element; + this.qualifier = null; + } + + public void setSchema(String schema) { + this.schema = schema; + + } + + public String getField() { + return schema + "." + element + (qualifier==null?"":("." + qualifier)); + } + + public String getElement() { + return element; + } + + public void setElement(String element) { + this.element = element; + } + + public String getQualifier() { + return qualifier; + } + + public void setQualifier(String qualifier) { + this.qualifier = qualifier; + } +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/MetadataFieldMapping.java b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/MetadataFieldMapping.java new file mode 100644 index 0000000000..00a47afdda --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/MetadataFieldMapping.java @@ -0,0 +1,26 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.importer.external.metadatamapping; + +import java.util.Collection; + +/** + * Created by Roeland Dillen (roeland at atmire dot com) + * Date: 18/09/12 + * Time: 14:41 + */ + +public interface MetadataFieldMapping { + + public MetadatumDTO toDCValue(MetadataFieldConfig field, String mf); + + public Collection resultToDCValueMapping(RecordType record); + + + +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/MetadatumDTO.java b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/MetadatumDTO.java new file mode 100644 index 0000000000..a5de8b717c --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/MetadatumDTO.java @@ -0,0 +1,56 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.importer.external.metadatamapping; + +/** + * Created by Philip Vissenaekens (philip at atmire dot com) + * Date: 21/10/15 + * Time: 09:52 + */ +public class MetadatumDTO { + + private String schema; + private String element; + private String qualifier; + private String value; + + public MetadatumDTO() { + } + + public String getSchema() { + return schema; + } + + public void setSchema(String schema) { + this.schema = schema; + } + + public String getElement() { + return element; + } + + public void setElement(String element) { + this.element = element; + } + + public String getQualifier() { + return qualifier; + } + + public void setQualifier(String qualifier) { + this.qualifier = qualifier; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/CombinedMetadatumContributor.java b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/CombinedMetadatumContributor.java new file mode 100644 index 0000000000..8a557a5023 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/CombinedMetadatumContributor.java @@ -0,0 +1,110 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.importer.external.metadatamapping.contributor; + +import org.dspace.importer.external.metadatamapping.MetadataFieldConfig; +import org.dspace.importer.external.metadatamapping.MetadataFieldMapping; +import org.dspace.importer.external.metadatamapping.MetadatumDTO; + +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; + +/** + * Created by Philip Vissenaekens (philip at atmire dot com) + * Date: 17/06/15 + * Time: 11:02 + */ +public class CombinedMetadatumContributor implements MetadataContributor { + private MetadataFieldConfig field; + + private LinkedList metadatumContributors; + + private String separator; + + private MetadataFieldMapping> metadataFieldMapping; + + public CombinedMetadatumContributor() { + } + + public CombinedMetadatumContributor(MetadataFieldConfig field, List metadatumContributors, String separator) { + this.field = field; + this.metadatumContributors = (LinkedList) metadatumContributors; + this.separator = separator; + } + + @Override + public void setMetadataFieldMapping(MetadataFieldMapping> metadataFieldMapping) { + this.metadataFieldMapping = metadataFieldMapping; + + for (MetadataContributor metadatumContributor : metadatumContributors) { + metadatumContributor.setMetadataFieldMapping(metadataFieldMapping); + } + } + + + + /** + * a separate Metadatum object is created for each index of Metadatum returned from the calls to + * MetadatumContributor.contributeMetadata(t) for each MetadatumContributor in the metadatumContributors list. + * We assume that each contributor returns the same amount of Metadatum objects + * @param t the object we are trying to translate + * @return + */ + @Override + public Collection contributeMetadata(T t) { + List values=new LinkedList(); + + LinkedList> metadatumLists = new LinkedList<>(); + + for (MetadataContributor metadatumContributor : metadatumContributors) { + LinkedList metadatums = (LinkedList) metadatumContributor.contributeMetadata(t); + metadatumLists.add(metadatums); + } + + for (int i = 0; i metadatums : metadatumLists) { + value.append(metadatums.get(i).getValue()); + + if(!metadatums.equals(metadatumLists.getLast())) { + value.append(separator); + } + } + values.add(metadataFieldMapping.toDCValue(field, value.toString())); + } + + return values; + } + + public MetadataFieldConfig getField() { + return field; + } + + public void setField(MetadataFieldConfig field) { + this.field = field; + } + + public LinkedList getMetadatumContributors() { + return metadatumContributors; + } + + public void setMetadatumContributors(LinkedList metadatumContributors) { + this.metadatumContributors = metadatumContributors; + } + + public String getSeparator() { + return separator; + } + + public void setSeparator(String separator) { + this.separator = separator; + } +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/MetadataContributor.java b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/MetadataContributor.java new file mode 100644 index 0000000000..f9dd2a43a6 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/MetadataContributor.java @@ -0,0 +1,25 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.importer.external.metadatamapping.contributor; + +import org.dspace.importer.external.metadatamapping.MetadataFieldMapping; +import org.dspace.importer.external.metadatamapping.MetadatumDTO; + +import java.util.Collection; + +/** + * Created by Roeland Dillen (roeland at atmire dot com) + * Date: 11/01/13 + * Time: 09:18 + */ +public interface MetadataContributor { + + public void setMetadataFieldMapping(MetadataFieldMapping> rt); + + public Collection contributeMetadata(RecordType t); +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/SimpleXpathMetadatumContributor.java b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/SimpleXpathMetadatumContributor.java new file mode 100644 index 0000000000..ac710026c5 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/SimpleXpathMetadatumContributor.java @@ -0,0 +1,112 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.importer.external.metadatamapping.contributor; + +import org.apache.axiom.om.OMAttribute; +import org.apache.axiom.om.OMElement; +import org.apache.axiom.om.OMText; +import org.apache.axiom.om.xpath.AXIOMXPath; +import org.dspace.importer.external.metadatamapping.MetadataFieldConfig; +import org.dspace.importer.external.metadatamapping.MetadataFieldMapping; +import org.dspace.importer.external.metadatamapping.MetadatumDTO; +import org.jaxen.JaxenException; +import org.springframework.beans.factory.annotation.Required; + +import javax.annotation.Resource; +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * Created by Roeland Dillen (roeland at atmire dot com) + * Date: 11/01/13 + * Time: 09:21 + */ +public class SimpleXpathMetadatumContributor implements MetadataContributor { + private MetadataFieldConfig field; + + public Map getPrefixToNamespaceMapping() { + return prefixToNamespaceMapping; + } + + private MetadataFieldMapping> metadataFieldMapping; + + public MetadataFieldMapping> getMetadataFieldMapping() { + return metadataFieldMapping; + } + + public void setMetadataFieldMapping(MetadataFieldMapping> metadataFieldMapping) { + this.metadataFieldMapping = metadataFieldMapping; + } + + @Resource(name="isiFullprefixMapping") + public void setPrefixToNamespaceMapping(Map prefixToNamespaceMapping) { + this.prefixToNamespaceMapping = prefixToNamespaceMapping; + } + + private Map prefixToNamespaceMapping; + + public SimpleXpathMetadatumContributor(String query, Map prefixToNamespaceMapping, MetadataFieldConfig field) { + this.query = query; + this.prefixToNamespaceMapping = prefixToNamespaceMapping; + this.field = field; + } + + public SimpleXpathMetadatumContributor() { + + } + + private String query; + + public MetadataFieldConfig getField() { + return field; + } + @Required + public void setField(MetadataFieldConfig field) { + this.field = field; + } + + public String getQuery() { + return query; + } + @Required + public void setQuery(String query) { + this.query = query; + } + + @Override + public Collection contributeMetadata(OMElement t) { + List values=new LinkedList(); + try { + AXIOMXPath xpath=new AXIOMXPath(query); + for(String ns:prefixToNamespaceMapping.keySet()){ + xpath.addNamespace(prefixToNamespaceMapping.get(ns),ns); + } + List nodes=xpath.selectNodes(t); + for(Object el:nodes) + if(el instanceof OMElement) + values.add(metadataFieldMapping.toDCValue(field, ((OMElement) el).getText())); + else if(el instanceof OMAttribute){ + values.add(metadataFieldMapping.toDCValue(field, ((OMAttribute) el).getAttributeValue())); + } else if(el instanceof String){ + values.add(metadataFieldMapping.toDCValue(field, (String) el)); + } else if(el instanceof OMText) + values.add(metadataFieldMapping.toDCValue(field, ((OMText) el).getText())); + else + { + System.err.println("node of type: "+el.getClass()); + } + return values; + } catch (JaxenException e) { + System.err.println(query); + throw new RuntimeException(e); + } + + } +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/processor/AuthorMetadataProcessorService.java b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/processor/AuthorMetadataProcessorService.java new file mode 100644 index 0000000000..c25021f3b6 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/processor/AuthorMetadataProcessorService.java @@ -0,0 +1,30 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.importer.external.metadatamapping.processor; + +import org.apache.commons.lang.StringUtils; +import org.dspace.importer.external.metadatamapping.service.MetadataProcessorService; + +/** + * Removes the last point from an author name, this is required for the SAP lookup + * + * User: kevin (kevin at atmire.com) + * Date: 23/10/12 + * Time: 09:50 + */ +public class AuthorMetadataProcessorService implements MetadataProcessorService { + + @Override + public String processMetadataValue(String value) { + String ret=value; + ret= StringUtils.strip(ret); + ret= StringUtils.stripEnd(ret, "."); + + return ret; + } +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/service/GenerateQueryService.java b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/service/GenerateQueryService.java new file mode 100644 index 0000000000..d0e050e062 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/service/GenerateQueryService.java @@ -0,0 +1,22 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.importer.external.metadatamapping.service; + +import org.dspace.content.Item; +import org.dspace.importer.external.Query; +import org.dspace.importer.external.MetadataSourceException; + +/** + * Created by Roeland Dillen (roeland at atmire dot com) + * Date: 14/12/12 + * Time: 11:44 + */ +public interface GenerateQueryService { + + public Query generateQueryForItem(Item item) throws MetadataSourceException; +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/service/MetadataProcessorService.java b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/service/MetadataProcessorService.java new file mode 100644 index 0000000000..85d54e86da --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/service/MetadataProcessorService.java @@ -0,0 +1,18 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.importer.external.metadatamapping.service; + +/** + * User: kevin (kevin at atmire.com) + * Date: 23/10/12 + * Time: 09:49 + */ +public interface MetadataProcessorService { + + public String processMetadataValue(String value); +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/AbstractImportMetadataSourceService.java b/dspace-api/src/main/java/org/dspace/importer/external/service/AbstractImportMetadataSourceService.java new file mode 100644 index 0000000000..4d7eed933a --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/AbstractImportMetadataSourceService.java @@ -0,0 +1,53 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ + +package org.dspace.importer.external.service; + +import org.dspace.importer.external.datamodel.ImportRecord; +import org.dspace.importer.external.metadatamapping.MetadataFieldMapping; +import org.dspace.importer.external.metadatamapping.MetadatumDTO; +import org.dspace.importer.external.metadatamapping.contributor.MetadataContributor; +import org.dspace.importer.external.metadatamapping.service.GenerateQueryService; +import org.dspace.importer.external.service.other.Imports; +import org.dspace.importer.external.service.other.MetadataSource; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Required; + +import java.util.LinkedList; + +/** + * Created by: Roeland Dillen (roeland at atmire dot com) + * Date: 29 May 2015 + */ +public abstract class AbstractImportMetadataSourceService extends MetadataSource implements Imports { + private GenerateQueryService generateQueryForItem = null; + private MetadataFieldMapping> metadataFieldMapping; + + public GenerateQueryService getGenerateQueryForItem() { + return generateQueryForItem; + } + + @Autowired + public void setGenerateQueryForItem(GenerateQueryService generateQueryForItem) { + this.generateQueryForItem = generateQueryForItem; + } + + public MetadataFieldMapping> getMetadataFieldMapping() { + return metadataFieldMapping; + } + + @Required + public void setMetadataFieldMapping( + MetadataFieldMapping> metadataFieldMapping) { + this.metadataFieldMapping = metadataFieldMapping; + } + + public ImportRecord transformSourceRecords(RecordType rt){ + return new ImportRecord(new LinkedList(getMetadataFieldMapping().resultToDCValueMapping(rt))); + } +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java b/dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java new file mode 100644 index 0000000000..7dd6e75b0b --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java @@ -0,0 +1,174 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ + +package org.dspace.importer.external.service; + +import org.apache.log4j.Logger; +import org.dspace.content.Item; +import org.dspace.importer.external.MetadataSourceException; +import org.dspace.importer.external.Query; +import org.dspace.importer.external.datamodel.ImportRecord; +import org.dspace.importer.external.service.other.Destroyable; +import org.dspace.importer.external.service.other.Imports; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.*; + +/** + * Created by Roeland Dillen (roeland at atmire dot com) + * Date: 17/09/12 + * Time: 14:19 + */ +public class ImportService implements Destroyable { + private HashMap importSources = new HashMap(); + + Logger log = Logger.getLogger(ImportService.class); + + public ImportService() { + + } + + protected static final String ANY = "*"; + + @Autowired(required = false) + public void setImportSources(List importSources) throws MetadataSourceException { + log.info("Loading " + importSources.size() + " import sources."); + for (Imports imports : importSources) { + this.importSources.put(imports.getImportSource(), imports); + } + + } + + protected Map getImportSources() { + return Collections.unmodifiableMap(importSources); + } + + protected Collection matchingImports(String url) { + if (ANY.equals(url)) { + return importSources.values(); + } else { + if(importSources.containsKey(url)) + return Collections.singletonList(importSources.get(url)); + else + return Collections.emptyList(); + } + } + + + public Collection findMatchingRecords(String url, Item item) throws MetadataSourceException { + try { + List recordList = new LinkedList(); + + for (Imports imports : matchingImports(url)) { + recordList.addAll(imports.findMatchingRecords(item)); + } + + return recordList; + } catch (Exception e) { + throw new MetadataSourceException(e); + } + } + + public Collection findMatchingRecords(String url, Query query) throws MetadataSourceException { + try { + List recordList = new LinkedList(); + for (Imports imports : matchingImports(url)) { + recordList.addAll(imports.findMatchingRecords(query)); + } + + return recordList; + } catch (Exception e) { + throw new MetadataSourceException(e); + } + } + + public int getNbRecords(String url, String query) throws MetadataSourceException { + try { + int total = 0; + for (Imports Imports : matchingImports(url)) { + total += Imports.getNbRecords(query); + } + return total; + } catch (Exception e) { + throw new MetadataSourceException(e); + } + } + + public int getNbRecords(String url, Query query) throws MetadataSourceException { + try { + int total = 0; + for (Imports Imports : matchingImports(url)) { + total += Imports.getNbRecords(query); + } + return total; + } catch (Exception e) { + throw new MetadataSourceException(e); + } + } + + + public Collection getRecords(String url, String query, int start, int count) throws MetadataSourceException { + try { + List recordList = new LinkedList(); + for (Imports imports : matchingImports(url)) { + recordList.addAll(imports.getRecords(query, start, count)); + } + return recordList; + } catch (Exception e) { + throw new MetadataSourceException(e); + } + } + public Collection getRecords(String url, Query query) throws MetadataSourceException { + try { + List recordList = new LinkedList(); + for (Imports imports : matchingImports(url)) { + recordList.addAll(imports.getRecords(query)); + } + return recordList; + } catch (Exception e) { + throw new MetadataSourceException(e); + } + } + + + public ImportRecord getRecord(String url, String id) throws MetadataSourceException { + try { + for (Imports imports : matchingImports(url)) { + if (imports.getRecord(id) != null) return imports.getRecord(id); + + } + return null; + } catch (Exception e) { + throw new MetadataSourceException(e); + } + } + + public ImportRecord getRecord(String url, Query query) throws MetadataSourceException { + try { + for (Imports imports : matchingImports(url)) { + if (imports.getRecord(query) != null) return imports.getRecord(query); + + } + return null; + } catch (Exception e) { + throw new MetadataSourceException(e); + } + } + + public Collection getImportUrls() { + return importSources.keySet(); + } + + + @Override + public void destroy() throws Exception { + for (Imports imports : importSources.values()) { + if (imports instanceof Destroyable) ((Destroyable) imports).destroy(); + } + } +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/other/Destroyable.java b/dspace-api/src/main/java/org/dspace/importer/external/service/other/Destroyable.java new file mode 100644 index 0000000000..b754a37557 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/other/Destroyable.java @@ -0,0 +1,18 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ + +package org.dspace.importer.external.service.other; + +/** + * Created by Roeland Dillen (roeland at atmire dot com) + * Date: 26/09/12 + * Time: 11:09 + */ +public interface Destroyable { + public void destroy() throws Exception; +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/other/Imports.java b/dspace-api/src/main/java/org/dspace/importer/external/service/other/Imports.java new file mode 100644 index 0000000000..84ee5bbbd2 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/other/Imports.java @@ -0,0 +1,35 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ + +package org.dspace.importer.external.service.other; + +import org.dspace.content.Item; +import org.dspace.importer.external.MetadataSourceException; +import org.dspace.importer.external.Query; +import org.dspace.importer.external.datamodel.ImportRecord; + +import java.util.Collection; + +/** + * Created by Roeland Dillen (roeland at atmire dot com) + * Date: 17/09/12 + * Time: 14:08 + */ +public interface Imports { + public int getNbRecords(String query) throws MetadataSourceException; + public int getNbRecords(Query query) throws MetadataSourceException; + public Collection getRecords(String query, int start, int count)throws MetadataSourceException; + public Collection getRecords(Query q)throws MetadataSourceException; + public ImportRecord getRecord(String id)throws MetadataSourceException; + public ImportRecord getRecord(Query q)throws MetadataSourceException; + public String getImportSource(); + + public Collection findMatchingRecords(Item item) throws MetadataSourceException; + + public Collection findMatchingRecords(Query q) throws MetadataSourceException; +} diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/other/MetadataSource.java b/dspace-api/src/main/java/org/dspace/importer/external/service/other/MetadataSource.java new file mode 100644 index 0000000000..7de892c3e8 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/other/MetadataSource.java @@ -0,0 +1,187 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ + +package org.dspace.importer.external.service.other; + +import org.apache.log4j.Logger; +import org.dspace.importer.external.MetadataSourceException; +import org.dspace.importer.external.SourceExceptionHandler; + +import javax.annotation.Resource; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.locks.ReentrantLock; + +/** + * Created by: Antoine Snyers (antoine at atmire dot com) + * Date: 27 Oct 2014 + */ +public abstract class MetadataSource { + + protected long lastRequest = 0; + protected long interRequestTime; + + protected ReentrantLock lock = new ReentrantLock(); + + protected int maxRetry = 20; + protected int retry; + protected String operationId; + protected String warning; + + protected Map> exceptionHandlersMap; + protected Exception error; + + + protected MetadataSource() { + initExceptionHandlers(); + } + + protected void initExceptionHandlers() { + exceptionHandlersMap = new LinkedHashMap>(); + // if an exception is thrown that is not in there, it is not recoverable and the retry chain will stop + // by default all exceptions are fatal, but subclasses can add their own handlers for their own exceptions + } + + public String getWarning() { + return warning; + } + + public void setWarning(String warning) { + this.warning = warning; + } + + public int getRetry() { + return retry; + } + + public int getMaxRetry() { + return maxRetry; + } + @Resource(name="maxRetry") + public void setMaxRetry(int maxRetry) { + this.maxRetry = maxRetry; + } + + public String getOperationId() { + return operationId; + } + + public Exception getError() { + return error; + } + + public void setError(Exception error) { + this.error = error; + } + + /** + * log4j logger + */ + private static Logger log = Logger.getLogger(MetadataSource.class); + + /** + * Command pattern implementation. the callable.call method will be retried + * until it either succeeds or reaches the try limit. Maybe this should have + * a backoff algorithm instead of waiting a fixed time. + * + * @param callable the callable to call. See the classes with the same name as + * the public methods of this class. + * @param return type. Generics for type safety. + * @return The result of the call + * @throws com.atmire.import_citations.configuration.SourceException if something unrecoverable happens (e.g. network failures) + */ + protected T retry(Callable callable) throws MetadataSourceException { + + retry = 0; + operationId = UUID.randomUUID().toString(); + while (true) { + try { + lock.lock(); + this.error = null; + long time = System.currentTimeMillis() - lastRequest; + if ((time) < interRequestTime) { + Thread.sleep(interRequestTime - time); + } + try { + init(); + } catch (Exception e) { + throwSourceException(retry, e, operationId); + } + log.info("operation " + operationId + " started"); + T response = callable.call(); + log.info("operation " + operationId + " successful"); + return response; + } catch (Exception e) { + this.error = e; + if (retry > maxRetry) { + throwSourceException(retry, e, operationId); + } + handleException(retry, e, operationId); + + // No MetadataSourceException has interrupted the loop + retry++; + log.warn("Error in trying operation " + operationId + " " + retry + " " + warning + ", retrying !", e); + + } finally { + lock.unlock(); + } + + try{ + Thread.sleep(1000L); + } catch (InterruptedException e) { + throwSourceException(retry, e, operationId); + } + + } + + } + + protected void handleException(int retry, Exception e, String operationId) throws MetadataSourceException { + + List exceptionHandlers = getExceptionHandler(e); + if (exceptionHandlers != null && !exceptionHandlers.isEmpty()) { + for (SourceExceptionHandler exceptionHandler : exceptionHandlers) { + exceptionHandler.handle(this); + } + }else{ + throwSourceException(retry, e, operationId); + } + } + + protected List getExceptionHandler(Exception e) { + for (Class aClass : exceptionHandlersMap.keySet()) { + if (aClass.isInstance(e)) { + return exceptionHandlersMap.get(aClass); + } + } + return null; + } + + protected void throwSourceException(int retry, Exception e, String operationId) throws MetadataSourceException { + throwSourceExceptionHook(); +// log.error("Source exception", e); + log.error("Source exception " + e.getMessage()); + throw new MetadataSourceException("At retry of operation " + operationId + " " + retry, e); + } + + protected void throwSourceExceptionHook() { + + } + + /** + * Attempts to init a session + * + * @throws Exception + */ + public abstract void init() throws Exception; + + +} diff --git a/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml b/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml new file mode 100644 index 0000000000..d227b08e7d --- /dev/null +++ b/dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml @@ -0,0 +1,25 @@ + + + + + + + + + \ No newline at end of file