29313: import citations framework integration into dspace6

29313: Integrate import citations framework into dspace6

29412: import external package refactoring

29412: import external package refactoring

more refactorying

dspace-api/pom.xml

@@ -670,6 +670,18 @@
             <version>1</version>
             <type>jar</type>
         </dependency>
+
+        <dependency>
+            <groupId>org.apache.ws.commons.axiom</groupId>
+            <artifactId>axiom-impl</artifactId>
+            <version>1.2.14</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.ws.commons.axiom</groupId>
+            <artifactId>axiom-api</artifactId>
+            <version>1.2.14</version>
+        </dependency>
         <!-- S3 -->
         <dependency>
             <groupId>com.amazonaws</groupId>

dspace-api/src/main/java/org/dspace/importer/external/MetadataSourceException.java

@@ -0,0 +1,23 @@
+package org.dspace.importer.external;
+
+/** Represents a problem with the input source: e.g. cannot connect to the source.
+ * Created by Roeland Dillen (roeland at atmire dot com)
+ * Date: 19/09/12
+ * Time: 13:17
+ */
+public class MetadataSourceException extends Exception {
+    public MetadataSourceException() {
+    }
+
+    public MetadataSourceException(String s) {
+        super(s);
+    }
+
+    public MetadataSourceException(String s, Throwable throwable) {
+        super(s, throwable);
+    }
+
+    public MetadataSourceException(Throwable throwable) {
+        super(throwable);
+    }
+}

dspace-api/src/main/java/org/dspace/importer/external/Query.java

@@ -0,0 +1,47 @@
+package org.dspace.importer.external;
+
+import org.apache.commons.collections.map.MultiValueMap;
+
+import java.util.Collection;
+
+/**
+ * Created by Roeland Dillen (roeland at atmire dot com)
+ * Date: 27/09/12
+ * Time: 15:26
+ */
+public class Query {
+    private MultiValueMap parameters = new MultiValueMap();
+
+    public MultiValueMap getParameters() {
+        return parameters;
+    }
+
+    public void addParameter(String key,Object value){
+        parameters.put(key,value);
+    }
+
+    protected void addSingletonParameter(String key,Object value){
+        parameters.remove(key);
+        parameters.put(key,value);
+    }
+
+    public <T> T getParameterAsClass(String key, Class<T> clazz){
+        Collection c=parameters.getCollection(key);
+        if(c==null||c.isEmpty()) return null;
+        else {
+            Object o=c.iterator().next();
+            if(clazz.isAssignableFrom(o.getClass()))
+            return (T) o ;
+            else return null;
+        }
+
+    }
+
+    public Collection getParameter(String key){
+        return parameters.getCollection(key);
+    }
+
+    public void setParameters(MultiValueMap parameters) {
+        this.parameters = parameters;
+    }
+}

dspace-api/src/main/java/org/dspace/importer/external/SourceExceptionHandler.java

@@ -0,0 +1,13 @@
+package org.dspace.importer.external;
+
+import org.dspace.importer.external.service.other.MetadataSource;
+
+/**
+ * Created by: Antoine Snyers (antoine at atmire dot com)
+ * Date: 27 Oct 2014
+ */
+public abstract interface SourceExceptionHandler<T extends MetadataSource> {
+
+    public abstract void handle(T source);
+
+}

dspace-api/src/main/java/org/dspace/importer/external/datamodel/ImportRecord.java

@@ -0,0 +1,68 @@
+package org.dspace.importer.external.datamodel;
+
+import org.dspace.importer.external.metadatamapping.MetadatumDTO;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * Created by Roeland Dillen (roeland at atmire dot com)
+ * Date: 17/09/12
+ * Time: 14:03
+ */
+public class ImportRecord {
+    private List<MetadatumDTO> valueList = null;
+
+    public List<MetadatumDTO> getValueList() {
+        return Collections.unmodifiableList(valueList);
+    }
+
+    public ImportRecord(List<MetadatumDTO> valueList) {
+        //don't want to alter the original list. Also now I can control the type of list
+        this.valueList = new LinkedList<MetadatumDTO>(valueList);
+    }
+
+    @Override
+    public String toString() {
+        final StringBuilder sb = new StringBuilder();
+        sb.append("Record");
+        sb.append("{valueList=");
+        for(MetadatumDTO val:valueList){
+            sb.append("{");
+            sb.append(val.getSchema());
+            sb.append("; ");
+            sb.append(val.getElement());
+            sb.append("; ");
+
+            sb.append(val.getQualifier());
+            sb.append("; ");
+
+            sb.append(val.getValue());
+            sb.append("; ");
+            sb.append("}\n");
+
+        }
+        sb.append("}\n");
+        return sb.toString();
+    }
+
+    public Collection<MetadatumDTO> getValue(String schema, String element, String qualifier){
+        List<MetadatumDTO> values=new LinkedList<MetadatumDTO>();
+        for(MetadatumDTO value:valueList){
+            if(value.getSchema().equals(schema)&&value.getElement().equals(element)){
+               if(qualifier==null&&value.getQualifier()==null){
+                   values.add(value);
+               } else if (value.getQualifier()!=null&&value.getQualifier().equals(qualifier)) {
+                   values.add(value);
+                }
+            }
+        }
+        return values;
+    }
+
+    public void addValue(MetadatumDTO value){
+        this.valueList.add(value);
+    }
+}

dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/AbstractMetadataFieldMapping.java

@@ -0,0 +1,120 @@
+package org.dspace.importer.external.metadatamapping;
+
+
+import org.apache.log4j.Logger;
+import org.dspace.importer.external.metadatamapping.contributor.MetadataContributor;
+import org.dspace.importer.external.metadatamapping.service.MetadataProcessorService;
+
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Created by Roeland Dillen (roeland at atmire dot com)
+ * Date: 19/09/12
+ * Time: 10:09
+ */
+public abstract class AbstractMetadataFieldMapping<RecordType> implements MetadataFieldMapping<RecordType, MetadataContributor<RecordType>> {
+
+    private Map<MetadataFieldConfig, MetadataContributor<RecordType>> metadataFieldMap;
+
+    /**
+     * log4j logger
+     */
+    private static Logger log = Logger.getLogger(AbstractMetadataFieldMapping.class);
+
+    private Map<MetadataFieldConfig, MetadataProcessorService> metadataProcessorMap;
+
+    public void setMetadataProcessorMap(Map<MetadataFieldConfig, MetadataProcessorService> metadataProcessorMap)
+    {
+        this.metadataProcessorMap = metadataProcessorMap;
+    }
+
+    public MetadataProcessorService getMetadataProcessor(MetadataFieldConfig metadataField)
+    {
+        if(metadataProcessorMap != null)
+        {
+            return metadataProcessorMap.get(metadataField);
+        }else{
+            return null;
+        }
+    }
+
+    public MetadatumDTO toDCValue(MetadataFieldConfig field, String value) {
+        MetadatumDTO dcValue = new MetadatumDTO();
+
+
+
+        if (field == null) return null;
+        MetadataProcessorService metadataProcessor = getMetadataProcessor(field);
+        if(metadataProcessor != null)
+        {
+            value = metadataProcessor.processMetadataValue(value);
+        }
+        dcValue.setValue(value);
+        dcValue.setElement(field.getElement());
+        dcValue.setQualifier(field.getQualifier());
+        dcValue.setSchema(field.getSchema());
+        return dcValue;
+    }
+
+    private boolean reverseDifferent = false;
+
+    private String AND = "AND";
+    private String OR = "OR";
+    private String NOT = "NOT";
+
+    public String getAND() {
+        return AND;
+    }
+
+    public void setAND(String AND) {
+        this.AND = AND;
+    }
+
+    public String getOR() {
+        return OR;
+    }
+
+    public void setOR(String OR) {
+        this.OR = OR;
+    }
+
+    public String getNOT() {
+        return NOT;
+    }
+
+    public void setNOT(String NOT) {
+        this.NOT = NOT;
+    }
+
+    public Map<MetadataFieldConfig, MetadataContributor<RecordType>> getMetadataFieldMap() {
+        return metadataFieldMap;
+    }
+
+    public void setMetadataFieldMap(Map<MetadataFieldConfig, MetadataContributor<RecordType>> metadataFieldMap) {
+        this.metadataFieldMap = metadataFieldMap;
+        for(MetadataContributor<RecordType> mc:metadataFieldMap.values()){
+            mc.setMetadataFieldMapping(this);
+        }
+
+    }
+
+    @Override
+    public Collection<MetadatumDTO> resultToDCValueMapping(RecordType record) {
+        List<MetadatumDTO> values=new LinkedList<MetadatumDTO>();
+
+
+        for(MetadataContributor<RecordType> query:getMetadataFieldMap().values()){
+            try {
+                values.addAll(query.contributeMetadata(record));
+            } catch (Exception e) {
+                log.error("Error", e);
+            }
+
+        }
+        return values;
+
+    }
+}

dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/MetadataFieldConfig.java

@@ -0,0 +1,110 @@
+package org.dspace.importer.external.metadatamapping;
+
+/**
+ * Created by Roeland Dillen (roeland at atmire dot com)
+ * Date: 19/09/12
+ * Time: 10:11
+ */
+public class MetadataFieldConfig {
+    private String schema;
+    private String element;
+    private String qualifier;
+
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        MetadataFieldConfig that = (MetadataFieldConfig) o;
+
+        if (!element.equals(that.element)) return false;
+        if (qualifier != null ? !qualifier.equals(that.qualifier) : that.qualifier != null) return false;
+        if (!schema.equals(that.schema)) return false;
+
+        return true;
+    }
+
+    @Override
+    public String toString() {
+        final StringBuilder sb = new StringBuilder();
+        sb.append("MetadataField");
+        sb.append("{schema='").append(schema).append('\'');
+        sb.append(", element='").append(element).append('\'');
+        sb.append(", qualifier='").append(qualifier).append('\'');
+        sb.append('}');
+        return sb.toString();
+    }
+
+    @Override
+    public int hashCode() {
+        int result = schema.hashCode();
+        result = 31 * result + element.hashCode();
+        result = 31 * result + (qualifier != null ? qualifier.hashCode() : 0);
+        return result;
+    }
+
+    public String getSchema() {
+
+        return schema;
+    }
+
+    public MetadataFieldConfig(MetadatumDTO value) {
+        this.schema = value.getSchema();
+        this.element = value.getElement();
+        this.qualifier = value.getQualifier();
+    }
+
+    public MetadataFieldConfig() {
+    }
+
+    public MetadataFieldConfig(String schema, String element, String qualifier) {
+        this.schema = schema;
+        this.element = element;
+        this.qualifier = qualifier;
+    }
+
+    public MetadataFieldConfig(String full) {
+        String elements[]=full.split("\\.");
+        if(elements.length==2){
+            this.schema = elements[0];
+            this.element =elements[1];
+        } else if(elements.length==3){
+            this.schema = elements[0];
+            this.element =elements[1];
+            this.qualifier = elements[2];
+        }
+
+    }
+
+    public MetadataFieldConfig(String schema, String element) {
+        this.schema = schema;
+        this.element = element;
+        this.qualifier = null;
+    }
+
+    public void setSchema(String schema) {
+        this.schema = schema;
+
+    }
+
+    public String getField() {
+        return schema + "." + element + (qualifier==null?"":("." + qualifier));
+    }
+
+    public String getElement() {
+        return element;
+    }
+
+    public void setElement(String element) {
+        this.element = element;
+    }
+
+    public String getQualifier() {
+        return qualifier;
+    }
+
+    public void setQualifier(String qualifier) {
+        this.qualifier = qualifier;
+    }
+}

dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/MetadataFieldMapping.java

@@ -0,0 +1,19 @@
+package org.dspace.importer.external.metadatamapping;
+
+import java.util.Collection;
+
+/**
+ * Created by Roeland Dillen (roeland at atmire dot com)
+ * Date: 18/09/12
+ * Time: 14:41
+ */
+
+public interface  MetadataFieldMapping<RecordType,QueryType> {
+
+        public MetadatumDTO toDCValue(MetadataFieldConfig field, String mf);
+
+        public Collection<MetadatumDTO> resultToDCValueMapping(RecordType record);
+
+
+
+}

dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/MetadatumDTO.java

@@ -0,0 +1,49 @@
+package org.dspace.importer.external.metadatamapping;
+
+/**
+ * Created by Philip Vissenaekens (philip at atmire dot com)
+ * Date: 21/10/15
+ * Time: 09:52
+ */
+public class MetadatumDTO {
+
+    private String schema;
+    private String element;
+    private String qualifier;
+    private String value;
+
+    public MetadatumDTO() {
+    }
+
+    public String getSchema() {
+        return schema;
+    }
+
+    public void setSchema(String schema) {
+        this.schema = schema;
+    }
+
+    public String getElement() {
+        return element;
+    }
+
+    public void setElement(String element) {
+        this.element = element;
+    }
+
+    public String getQualifier() {
+        return qualifier;
+    }
+
+    public void setQualifier(String qualifier) {
+        this.qualifier = qualifier;
+    }
+
+    public String getValue() {
+        return value;
+    }
+
+    public void setValue(String value) {
+        this.value = value;
+    }
+}

dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/CombinedMetadatumContributor.java

@@ -0,0 +1,103 @@
+package org.dspace.importer.external.metadatamapping.contributor;
+
+import org.dspace.importer.external.metadatamapping.MetadataFieldConfig;
+import org.dspace.importer.external.metadatamapping.MetadataFieldMapping;
+import org.dspace.importer.external.metadatamapping.MetadatumDTO;
+
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * Created by Philip Vissenaekens (philip at atmire dot com)
+ * Date: 17/06/15
+ * Time: 11:02
+ */
+public class CombinedMetadatumContributor<T> implements MetadataContributor<T> {
+    private MetadataFieldConfig field;
+
+    private LinkedList<MetadataContributor> metadatumContributors;
+
+    private String separator;
+
+    private MetadataFieldMapping<T,MetadataContributor<T>> metadataFieldMapping;
+
+    public CombinedMetadatumContributor() {
+    }
+
+    public CombinedMetadatumContributor(MetadataFieldConfig field, List<MetadataContributor> metadatumContributors, String separator) {
+        this.field = field;
+        this.metadatumContributors = (LinkedList<MetadataContributor>) metadatumContributors;
+        this.separator = separator;
+    }
+
+    @Override
+    public void setMetadataFieldMapping(MetadataFieldMapping<T, MetadataContributor<T>> metadataFieldMapping) {
+        this.metadataFieldMapping = metadataFieldMapping;
+
+        for (MetadataContributor metadatumContributor : metadatumContributors) {
+            metadatumContributor.setMetadataFieldMapping(metadataFieldMapping);
+        }
+    }
+
+
+
+    /**
+     * a separate Metadatum object is created for each index of Metadatum returned from the calls to
+     * MetadatumContributor.contributeMetadata(t) for each MetadatumContributor in the metadatumContributors list.
+     * We assume that each contributor returns the same amount of Metadatum objects
+     * @param t the object we are trying to translate
+     * @return
+     */
+    @Override
+    public Collection<MetadatumDTO> contributeMetadata(T t) {
+        List<MetadatumDTO> values=new LinkedList<MetadatumDTO>();
+
+        LinkedList<LinkedList<MetadatumDTO>> metadatumLists = new LinkedList<>();
+
+        for (MetadataContributor metadatumContributor : metadatumContributors) {
+            LinkedList<MetadatumDTO> metadatums = (LinkedList<MetadatumDTO>) metadatumContributor.contributeMetadata(t);
+            metadatumLists.add(metadatums);
+        }
+
+        for (int i = 0; i<metadatumLists.getFirst().size();i++) {
+
+            StringBuilder value = new StringBuilder();
+
+            for (LinkedList<MetadatumDTO> metadatums : metadatumLists) {
+                value.append(metadatums.get(i).getValue());
+
+                if(!metadatums.equals(metadatumLists.getLast())) {
+                    value.append(separator);
+                }
+            }
+            values.add(metadataFieldMapping.toDCValue(field, value.toString()));
+        }
+
+        return values;
+    }
+
+    public MetadataFieldConfig getField() {
+        return field;
+    }
+
+    public void setField(MetadataFieldConfig field) {
+        this.field = field;
+    }
+
+    public LinkedList<MetadataContributor> getMetadatumContributors() {
+        return metadatumContributors;
+    }
+
+    public void setMetadatumContributors(LinkedList<MetadataContributor> metadatumContributors) {
+        this.metadatumContributors = metadatumContributors;
+    }
+
+    public String getSeparator() {
+        return separator;
+    }
+
+    public void setSeparator(String separator) {
+        this.separator = separator;
+    }
+}

dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/MetadataContributor.java

@@ -0,0 +1,18 @@
+package org.dspace.importer.external.metadatamapping.contributor;
+
+import org.dspace.importer.external.metadatamapping.MetadataFieldMapping;
+import org.dspace.importer.external.metadatamapping.MetadatumDTO;
+
+import java.util.Collection;
+
+/**
+ * Created by Roeland Dillen (roeland at atmire dot com)
+ * Date: 11/01/13
+ * Time: 09:18
+ */
+public interface MetadataContributor<RecordType> {
+
+    public void setMetadataFieldMapping(MetadataFieldMapping<RecordType, MetadataContributor<RecordType>> rt);
+
+    public Collection<MetadatumDTO> contributeMetadata(RecordType t);
+}

dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/SimpleXpathMetadatumContributor.java

@@ -0,0 +1,105 @@
+package org.dspace.importer.external.metadatamapping.contributor;
+
+import org.apache.axiom.om.OMAttribute;
+import org.apache.axiom.om.OMElement;
+import org.apache.axiom.om.OMText;
+import org.apache.axiom.om.xpath.AXIOMXPath;
+import org.dspace.importer.external.metadatamapping.MetadataFieldConfig;
+import org.dspace.importer.external.metadatamapping.MetadataFieldMapping;
+import org.dspace.importer.external.metadatamapping.MetadatumDTO;
+import org.jaxen.JaxenException;
+import org.springframework.beans.factory.annotation.Required;
+
+import javax.annotation.Resource;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Created by Roeland Dillen (roeland at atmire dot com)
+ * Date: 11/01/13
+ * Time: 09:21
+ */
+public class SimpleXpathMetadatumContributor implements MetadataContributor<OMElement> {
+    private MetadataFieldConfig field;
+
+    public Map<String, String> getPrefixToNamespaceMapping() {
+        return prefixToNamespaceMapping;
+    }
+
+    private MetadataFieldMapping<OMElement,MetadataContributor<OMElement>> metadataFieldMapping;
+
+    public MetadataFieldMapping<OMElement,MetadataContributor<OMElement>> getMetadataFieldMapping() {
+        return metadataFieldMapping;
+    }
+
+    public void setMetadataFieldMapping(MetadataFieldMapping<OMElement,MetadataContributor<OMElement>> metadataFieldMapping) {
+        this.metadataFieldMapping = metadataFieldMapping;
+    }
+
+    @Resource(name="isiFullprefixMapping")
+    public void setPrefixToNamespaceMapping(Map<String, String> prefixToNamespaceMapping) {
+        this.prefixToNamespaceMapping = prefixToNamespaceMapping;
+    }
+
+    private Map<String,String> prefixToNamespaceMapping;
+
+    public SimpleXpathMetadatumContributor(String query, Map<String, String> prefixToNamespaceMapping, MetadataFieldConfig field) {
+        this.query = query;
+        this.prefixToNamespaceMapping = prefixToNamespaceMapping;
+        this.field = field;
+    }
+
+    public SimpleXpathMetadatumContributor() {
+
+    }
+
+    private String query;
+
+    public MetadataFieldConfig getField() {
+        return field;
+    }
+    @Required
+    public void setField(MetadataFieldConfig field) {
+        this.field = field;
+    }
+
+    public String getQuery() {
+        return query;
+    }
+    @Required
+    public void setQuery(String query) {
+        this.query = query;
+    }
+
+    @Override
+    public Collection<MetadatumDTO> contributeMetadata(OMElement t) {
+        List<MetadatumDTO> values=new LinkedList<MetadatumDTO>();
+        try {
+            AXIOMXPath xpath=new AXIOMXPath(query);
+            for(String ns:prefixToNamespaceMapping.keySet()){
+                xpath.addNamespace(prefixToNamespaceMapping.get(ns),ns);
+            }
+            List<Object> nodes=xpath.selectNodes(t);
+            for(Object el:nodes)
+                if(el instanceof OMElement)
+                    values.add(metadataFieldMapping.toDCValue(field, ((OMElement) el).getText()));
+                else if(el instanceof OMAttribute){
+                    values.add(metadataFieldMapping.toDCValue(field, ((OMAttribute) el).getAttributeValue()));
+                } else if(el instanceof String){
+                    values.add(metadataFieldMapping.toDCValue(field, (String) el));
+                } else if(el instanceof OMText)
+                    values.add(metadataFieldMapping.toDCValue(field, ((OMText) el).getText()));
+                else
+                {
+                    System.err.println("node of type: "+el.getClass());
+                }
+            return values;
+        } catch (JaxenException e) {
+            System.err.println(query);
+            throw new RuntimeException(e);
+        }
+
+    }
+}

dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/processor/AuthorMetadataProcessorService.java

@@ -0,0 +1,23 @@
+package org.dspace.importer.external.metadatamapping.processor;
+
+import org.apache.commons.lang.StringUtils;
+import org.dspace.importer.external.metadatamapping.service.MetadataProcessorService;
+
+/**
+ * Removes the last point from an author name, this is required for the SAP lookup
+ *
+ * User: kevin (kevin at atmire.com)
+ * Date: 23/10/12
+ * Time: 09:50
+ */
+public class AuthorMetadataProcessorService implements MetadataProcessorService {
+
+    @Override
+    public String processMetadataValue(String value) {
+        String ret=value;
+        ret= StringUtils.strip(ret);
+        ret= StringUtils.stripEnd(ret, ".");
+
+        return ret;
+    }
+}

dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/service/GenerateQueryService.java

@@ -0,0 +1,15 @@
+package org.dspace.importer.external.metadatamapping.service;
+
+import org.dspace.content.Item;
+import org.dspace.importer.external.Query;
+import org.dspace.importer.external.MetadataSourceException;
+
+/**
+ * Created by Roeland Dillen (roeland at atmire dot com)
+ * Date: 14/12/12
+ * Time: 11:44
+ */
+public interface GenerateQueryService {
+
+    public Query generateQueryForItem(Item item) throws MetadataSourceException;
+}

dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/service/MetadataProcessorService.java

@@ -0,0 +1,11 @@
+package org.dspace.importer.external.metadatamapping.service;
+
+/**
+ * User: kevin (kevin at atmire.com)
+ * Date: 23/10/12
+ * Time: 09:49
+ */
+public interface MetadataProcessorService {
+
+    public String processMetadataValue(String value);
+}

dspace-api/src/main/java/org/dspace/importer/external/service/AbstractImportMetadataSourceService.java

@@ -0,0 +1,45 @@
+package org.dspace.importer.external.service;
+
+import org.dspace.importer.external.datamodel.ImportRecord;
+import org.dspace.importer.external.metadatamapping.MetadataFieldMapping;
+import org.dspace.importer.external.metadatamapping.MetadatumDTO;
+import org.dspace.importer.external.metadatamapping.contributor.MetadataContributor;
+import org.dspace.importer.external.metadatamapping.service.GenerateQueryService;
+import org.dspace.importer.external.service.other.Imports;
+import org.dspace.importer.external.service.other.MetadataSource;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Required;
+
+import java.util.LinkedList;
+
+/**
+ * Created by: Roeland Dillen (roeland at atmire dot com)
+ * Date: 29 May 2015
+ */
+public abstract class AbstractImportMetadataSourceService<RecordType> extends MetadataSource implements Imports {
+	private GenerateQueryService generateQueryForItem = null;
+	private MetadataFieldMapping<RecordType, MetadataContributor<RecordType>> metadataFieldMapping;
+
+	public GenerateQueryService getGenerateQueryForItem() {
+		return generateQueryForItem;
+	}
+
+	@Autowired
+	public void setGenerateQueryForItem(GenerateQueryService generateQueryForItem) {
+		this.generateQueryForItem = generateQueryForItem;
+	}
+
+	public MetadataFieldMapping<RecordType, MetadataContributor<RecordType>> getMetadataFieldMapping() {
+		return metadataFieldMapping;
+	}
+
+	@Required
+	public void setMetadataFieldMapping(
+			MetadataFieldMapping<RecordType, MetadataContributor<RecordType>> metadataFieldMapping) {
+		this.metadataFieldMapping = metadataFieldMapping;
+	}
+
+	public ImportRecord transformSourceRecords(RecordType rt){
+		 return new ImportRecord(new LinkedList<MetadatumDTO>(getMetadataFieldMapping().resultToDCValueMapping(rt)));
+	}
+}

dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java

@@ -0,0 +1,166 @@
+package org.dspace.importer.external.service;
+
+import org.apache.log4j.Logger;
+import org.dspace.content.Item;
+import org.dspace.importer.external.MetadataSourceException;
+import org.dspace.importer.external.Query;
+import org.dspace.importer.external.datamodel.ImportRecord;
+import org.dspace.importer.external.service.other.Destroyable;
+import org.dspace.importer.external.service.other.Imports;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.*;
+
+/**
+ * Created by Roeland Dillen (roeland at atmire dot com)
+ * Date: 17/09/12
+ * Time: 14:19
+ */
+public class ImportService implements Destroyable {
+    private HashMap<String, Imports> importSources = new HashMap<String, Imports>();
+
+    Logger log = Logger.getLogger(ImportService.class);
+
+    public ImportService() {
+
+    }
+
+    protected static final String ANY = "*";
+
+    @Autowired(required = false)
+    public void setImportSources(List<Imports> importSources) throws MetadataSourceException {
+        log.info("Loading " + importSources.size() + " import sources.");
+        for (Imports imports : importSources) {
+            this.importSources.put(imports.getImportSource(), imports);
+        }
+
+    }
+
+    protected Map<String, Imports> getImportSources() {
+        return Collections.unmodifiableMap(importSources);
+    }
+
+    protected Collection<Imports> matchingImports(String url) {
+        if (ANY.equals(url)) {
+            return importSources.values();
+        } else {
+			if(importSources.containsKey(url))
+				return Collections.singletonList(importSources.get(url));
+			else
+				return Collections.emptyList();
+		}
+    }
+
+
+    public Collection<ImportRecord> findMatchingRecords(String url, Item item) throws MetadataSourceException {
+		try {
+			List<ImportRecord> recordList = new LinkedList<ImportRecord>();
+
+			for (Imports imports : matchingImports(url)) {
+				recordList.addAll(imports.findMatchingRecords(item));
+			}
+
+			return recordList;
+		} catch (Exception e) {
+			throw new MetadataSourceException(e);
+		}
+	}
+
+    public Collection<ImportRecord> findMatchingRecords(String url, Query query) throws MetadataSourceException {
+		try {
+			List<ImportRecord> recordList = new LinkedList<ImportRecord>();
+			for (Imports imports : matchingImports(url)) {
+				recordList.addAll(imports.findMatchingRecords(query));
+			}
+
+			return recordList;
+		} catch (Exception e) {
+			throw new MetadataSourceException(e);
+		}
+	}
+
+    public int getNbRecords(String url, String query) throws MetadataSourceException {
+		try {
+			int total = 0;
+			for (Imports Imports : matchingImports(url)) {
+				total += Imports.getNbRecords(query);
+			}
+			return total;
+		} catch (Exception e) {
+			throw new MetadataSourceException(e);
+		}
+	}
+
+    public int getNbRecords(String url, Query query) throws MetadataSourceException {
+		try {
+			int total = 0;
+			for (Imports Imports : matchingImports(url)) {
+				total += Imports.getNbRecords(query);
+			}
+			return total;
+		} catch (Exception e) {
+			throw new MetadataSourceException(e);
+		}
+	}
+
+
+    public Collection<ImportRecord> getRecords(String url, String query, int start, int count) throws MetadataSourceException {
+		try {
+			List<ImportRecord> recordList = new LinkedList<ImportRecord>();
+			for (Imports imports : matchingImports(url)) {
+				recordList.addAll(imports.getRecords(query, start, count));
+			}
+			return recordList;
+		} catch (Exception e) {
+			throw new MetadataSourceException(e);
+		}
+	}
+    public Collection<ImportRecord> getRecords(String url, Query query) throws MetadataSourceException {
+		try {
+			List<ImportRecord> recordList = new LinkedList<ImportRecord>();
+			for (Imports imports : matchingImports(url)) {
+				recordList.addAll(imports.getRecords(query));
+			}
+			return recordList;
+		} catch (Exception e) {
+			throw new MetadataSourceException(e);
+		}
+	}
+
+
+    public ImportRecord getRecord(String url, String id) throws MetadataSourceException {
+		try {
+			for (Imports imports : matchingImports(url)) {
+				if (imports.getRecord(id) != null) return imports.getRecord(id);
+	
+			}
+			return null;
+		} catch (Exception e) {
+			throw new MetadataSourceException(e);
+		}
+	}
+
+    public ImportRecord getRecord(String url, Query query) throws MetadataSourceException {
+		try {
+			for (Imports imports : matchingImports(url)) {
+				if (imports.getRecord(query) != null) return imports.getRecord(query);
+	
+			}
+			return null;
+		} catch (Exception e) {
+			throw new MetadataSourceException(e);
+		}
+	}
+
+    public Collection<String> getImportUrls() {
+        return importSources.keySet();
+    }
+
+
+    @Override
+    public void destroy() throws Exception {
+        for (Imports imports : importSources.values()) {
+            if (imports instanceof Destroyable) ((Destroyable) imports).destroy();
+        }
+    }
+}

dspace-api/src/main/java/org/dspace/importer/external/service/other/Destroyable.java

@@ -0,0 +1,10 @@
+package org.dspace.importer.external.service.other;
+
+/**
+ * Created by Roeland Dillen (roeland at atmire dot com)
+ * Date: 26/09/12
+ * Time: 11:09
+ */
+public interface Destroyable {
+    public void destroy() throws Exception;
+}

dspace-api/src/main/java/org/dspace/importer/external/service/other/Imports.java

@@ -0,0 +1,27 @@
+package org.dspace.importer.external.service.other;
+
+import org.dspace.content.Item;
+import org.dspace.importer.external.MetadataSourceException;
+import org.dspace.importer.external.Query;
+import org.dspace.importer.external.datamodel.ImportRecord;
+
+import java.util.Collection;
+
+/**
+ * Created by Roeland Dillen (roeland at atmire dot com)
+ * Date: 17/09/12
+ * Time: 14:08
+ */
+public interface Imports {
+    public int getNbRecords(String query) throws MetadataSourceException;
+    public int getNbRecords(Query query) throws MetadataSourceException;
+    public Collection<ImportRecord> getRecords(String query, int start, int count)throws MetadataSourceException;
+    public Collection<ImportRecord> getRecords(Query q)throws MetadataSourceException;
+    public ImportRecord getRecord(String id)throws MetadataSourceException;
+    public ImportRecord getRecord(Query q)throws MetadataSourceException;
+    public String getImportSource();
+
+    public Collection<ImportRecord> findMatchingRecords(Item item) throws MetadataSourceException;
+
+    public Collection<ImportRecord> findMatchingRecords(Query q) throws MetadataSourceException;
+}

dspace-api/src/main/java/org/dspace/importer/external/service/other/MetadataSource.java

@@ -0,0 +1,179 @@
+package org.dspace.importer.external.service.other;
+
+import org.apache.log4j.Logger;
+import org.dspace.importer.external.MetadataSourceException;
+import org.dspace.importer.external.SourceExceptionHandler;
+
+import javax.annotation.Resource;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * Created by: Antoine Snyers (antoine at atmire dot com)
+ * Date: 27 Oct 2014
+ */
+public abstract class MetadataSource {
+
+    protected long lastRequest = 0;
+    protected long interRequestTime;
+
+    protected ReentrantLock lock = new ReentrantLock();
+
+    protected int maxRetry = 20;
+    protected int retry;
+    protected String operationId;
+    protected String warning;
+
+    protected Map<Class, List<SourceExceptionHandler>> exceptionHandlersMap;
+    protected Exception error;
+
+
+    protected MetadataSource() {
+        initExceptionHandlers();
+    }
+
+    protected void initExceptionHandlers() {
+        exceptionHandlersMap = new LinkedHashMap<Class, List<SourceExceptionHandler>>();
+        // if an exception is thrown that is not in there, it is not recoverable and the retry chain will stop
+        // by default all exceptions are fatal, but subclasses can add their own handlers for their own exceptions
+    }
+
+    public String getWarning() {
+        return warning;
+    }
+
+    public void setWarning(String warning) {
+        this.warning = warning;
+    }
+
+    public int getRetry() {
+        return retry;
+    }
+
+    public int getMaxRetry() {
+        return maxRetry;
+    }
+    @Resource(name="maxRetry")
+    public void setMaxRetry(int maxRetry) {
+        this.maxRetry = maxRetry;
+    }
+
+    public String getOperationId() {
+        return operationId;
+    }
+
+    public Exception getError() {
+        return error;
+    }
+
+    public void setError(Exception error) {
+        this.error = error;
+    }
+
+    /**
+     * log4j logger
+     */
+    private static Logger log = Logger.getLogger(MetadataSource.class);
+
+    /**
+     * Command pattern implementation. the callable.call method will be retried
+     * until it either succeeds or reaches the try limit. Maybe this should have
+     * a backoff algorithm instead of waiting a fixed time.
+     *
+     * @param callable the callable to call. See the classes with the same name as
+     *                 the public methods of this class.
+     * @param <T>      return type. Generics for type safety.
+     * @return The result of the call
+     * @throws com.atmire.import_citations.configuration.SourceException if something unrecoverable happens (e.g. network failures)
+     */
+    protected <T> T retry(Callable<T> callable) throws MetadataSourceException {
+
+        retry = 0;
+        operationId = UUID.randomUUID().toString();
+        while (true) {
+            try {
+                lock.lock();
+                this.error = null;
+                long time = System.currentTimeMillis() - lastRequest;
+                if ((time) < interRequestTime) {
+                    Thread.sleep(interRequestTime - time);
+                }
+                try {
+                    init();
+                } catch (Exception e) {
+                    throwSourceException(retry, e, operationId);
+                }
+                log.info("operation " + operationId + " started");
+                T response = callable.call();
+                log.info("operation " + operationId + " successful");
+                return response;
+            } catch (Exception e) {
+                this.error = e;
+                if (retry > maxRetry) {
+                    throwSourceException(retry, e, operationId);
+                }
+                handleException(retry, e, operationId);
+
+                // No MetadataSourceException has interrupted the loop
+                retry++;
+                log.warn("Error in trying operation " + operationId + " " + retry + " " + warning + ", retrying !", e);
+
+            } finally {
+                lock.unlock();
+            }
+
+            try{
+                Thread.sleep(1000L);
+            } catch (InterruptedException e) {
+                throwSourceException(retry, e, operationId);
+            }
+
+        }
+
+    }
+
+    protected void handleException(int retry, Exception e, String operationId) throws MetadataSourceException {
+
+        List<SourceExceptionHandler> exceptionHandlers = getExceptionHandler(e);
+        if (exceptionHandlers != null && !exceptionHandlers.isEmpty()) {
+            for (SourceExceptionHandler exceptionHandler : exceptionHandlers) {
+                exceptionHandler.handle(this);
+            }
+        }else{
+            throwSourceException(retry, e, operationId);
+        }
+    }
+
+    protected List<SourceExceptionHandler> getExceptionHandler(Exception e) {
+        for (Class aClass : exceptionHandlersMap.keySet()) {
+            if (aClass.isInstance(e)) {
+                return exceptionHandlersMap.get(aClass);
+            }
+        }
+        return null;
+    }
+
+    protected void throwSourceException(int retry, Exception e, String operationId) throws MetadataSourceException {
+        throwSourceExceptionHook();
+//        log.error("Source exception", e);
+        log.error("Source exception " + e.getMessage());
+        throw new MetadataSourceException("At retry of operation " + operationId + " " + retry, e);
+    }
+
+    protected void throwSourceExceptionHook() {
+
+    }
+
+    /**
+     * Attempts to init a session
+     *
+     * @throws Exception
+     */
+    public abstract void init() throws Exception;
+
+
+}

dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    The contents of this file are subject to the license and copyright
+    detailed in the LICENSE and NOTICE files at the root of the source
+    tree and available online at
+
+    http://www.dspace.org/license/
+
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns:context="http://www.springframework.org/schema/context"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans
+           http://www.springframework.org/schema/beans/spring-beans.xsd
+           http://www.springframework.org/schema/context
+           http://www.springframework.org/schema/context/spring-context.xsd"
+       default-autowire-candidates="*Service,*DAO,javax.sql.DataSource">
+
+    <context:annotation-config /> <!-- allows us to use spring annotations in beans -->
+
+    <bean id="importService" class="org.dspace.importer.external.service.ImportService" scope="singleton"
+          lazy-init="false" autowire="byType" destroy-method="destroy"/>
+
+</beans>