From 1050d02a97cbe0ec83336f8bb1c8906f88aad62c Mon Sep 17 00:00:00 2001 From: Mykhaylo Date: Sat, 2 Apr 2022 13:11:11 +0200 Subject: [PATCH] refactored live import client and some utils class --- .../scopus/service/LiveImportClient.java | 3 +- .../scopus/service/LiveImportClientImpl.java | 56 ++++++++----- .../importer/external/service/DoiCheck.java | 52 ++++++++++++ .../config/spring/api/external-services.xml | 2 + .../AbstractLiveImportIntegrationTest.java | 80 +++++++++++++++++++ 5 files changed, 172 insertions(+), 21 deletions(-) create mode 100644 dspace-api/src/main/java/org/dspace/importer/external/service/DoiCheck.java create mode 100644 dspace-server-webapp/src/test/java/org/dspace/app/rest/AbstractLiveImportIntegrationTest.java diff --git a/dspace-api/src/main/java/org/dspace/importer/external/scopus/service/LiveImportClient.java b/dspace-api/src/main/java/org/dspace/importer/external/scopus/service/LiveImportClient.java index 50006fd486..bbc69c2a3d 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/scopus/service/LiveImportClient.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/scopus/service/LiveImportClient.java @@ -7,7 +7,6 @@ */ package org.dspace.importer.external.scopus.service; -import java.io.InputStream; import java.util.Map; /** @@ -17,6 +16,6 @@ import java.util.Map; */ public interface LiveImportClient { - public InputStream executeHttpGetRequest(int timeout, String URL, Map requestParams); + public String executeHttpGetRequest(int timeout, String URL, Map requestParams); } \ No newline at end of file diff --git a/dspace-api/src/main/java/org/dspace/importer/external/scopus/service/LiveImportClientImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/scopus/service/LiveImportClientImpl.java index f11e2fc4f2..d5503831d3 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/scopus/service/LiveImportClientImpl.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/scopus/service/LiveImportClientImpl.java @@ -9,20 +9,21 @@ package org.dspace.importer.external.scopus.service; import java.io.InputStream; import java.net.URISyntaxException; +import java.nio.charset.Charset; import java.util.Map; import java.util.Objects; +import java.util.Optional; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; -import org.apache.http.client.HttpClient; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.config.RequestConfig.Builder; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.utils.URIBuilder; -import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; -import org.apache.http.impl.conn.DefaultProxyRoutePlanner; import org.apache.log4j.Logger; import org.dspace.services.ConfigurationService; import org.springframework.beans.factory.annotation.Autowired; @@ -36,34 +37,32 @@ public class LiveImportClientImpl implements LiveImportClient { private static final Logger log = Logger.getLogger(LiveImportClientImpl.class); + private CloseableHttpClient httpClient; + @Autowired private ConfigurationService configurationService; @Override - public InputStream executeHttpGetRequest(int timeout, String URL, Map requestParams) { + public String executeHttpGetRequest(int timeout, String URL, Map requestParams) { HttpGet method = null; - String proxyHost = configurationService.getProperty("http.proxy.host"); - String proxyPort = configurationService.getProperty("http.proxy.port"); - try { - HttpClientBuilder hcBuilder = HttpClients.custom(); + try (CloseableHttpClient httpClient = Optional.ofNullable(this.httpClient) + .orElseGet(HttpClients::createDefault)) { + Builder requestConfigBuilder = RequestConfig.custom(); requestConfigBuilder.setConnectionRequestTimeout(timeout); - - if (StringUtils.isNotBlank(proxyHost) && StringUtils.isNotBlank(proxyPort)) { - HttpHost proxy = new HttpHost(proxyHost, Integer.parseInt(proxyPort), "http"); - DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy); - hcBuilder.setRoutePlanner(routePlanner); - } + RequestConfig defaultRequestConfig = requestConfigBuilder.build(); method = new HttpGet(getSearchUrl(URL, requestParams)); - method.setConfig(requestConfigBuilder.build()); + method.setConfig(defaultRequestConfig); - HttpClient client = hcBuilder.build(); - HttpResponse httpResponse = client.execute(method); + configureProxy(method, defaultRequestConfig); + + HttpResponse httpResponse = httpClient.execute(method); if (isNotSuccessfull(httpResponse)) { throw new RuntimeException(); } - return httpResponse.getEntity().getContent(); + InputStream inputStream = httpResponse.getEntity().getContent(); + return IOUtils.toString(inputStream, Charset.defaultCharset()); } catch (Exception e1) { log.error(e1.getMessage(), e1); } finally { @@ -71,7 +70,18 @@ public class LiveImportClientImpl implements LiveImportClient { method.releaseConnection(); } } - return null; + return StringUtils.EMPTY; + } + + private void configureProxy(HttpGet method, RequestConfig defaultRequestConfig) { + String proxyHost = configurationService.getProperty("http.proxy.host"); + String proxyPort = configurationService.getProperty("http.proxy.port"); + if (StringUtils.isNotBlank(proxyHost) && StringUtils.isNotBlank(proxyPort)) { + RequestConfig requestConfig = RequestConfig.copy(defaultRequestConfig) + .setProxy(new HttpHost(proxyHost, Integer.parseInt(proxyPort), "http")) + .build(); + method.setConfig(requestConfig); + } } private String getSearchUrl(String URL, Map requestParams) throws URISyntaxException { @@ -91,4 +101,12 @@ public class LiveImportClientImpl implements LiveImportClient { return response.getStatusLine().getStatusCode(); } + public CloseableHttpClient getHttpClient() { + return httpClient; + } + + public void setHttpClient(CloseableHttpClient httpClient) { + this.httpClient = httpClient; + } + } \ No newline at end of file diff --git a/dspace-api/src/main/java/org/dspace/importer/external/service/DoiCheck.java b/dspace-api/src/main/java/org/dspace/importer/external/service/DoiCheck.java new file mode 100644 index 0000000000..1285d6ea32 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/importer/external/service/DoiCheck.java @@ -0,0 +1,52 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.importer.external.service; + +import java.util.Arrays; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + + +/** + * Utility class that provides methods to check if a given string is a DOI and exists on CrossRef services + * + * @author Corrado Lombardi (corrado.lombardi at 4science.it) + */ +public class DoiCheck { + + private static final List DOI_PREFIXES = Arrays.asList("http://dx.doi.org/", "https://dx.doi.org/"); + + private static final Pattern PATTERN = Pattern.compile("10.\\d{4,9}/[-._;()/:A-Z0-9]+" + + "|10.1002/[^\\s]+" + + "|10.\\d{4}/\\d+-\\d+X?(\\d+)" + + "\\d+<[\\d\\w]+:[\\d\\w]*>\\d+.\\d+.\\w+;\\d" + + "|10.1021/\\w\\w\\d++" + + "|10.1207/[\\w\\d]+\\&\\d+_\\d+", + Pattern.CASE_INSENSITIVE); + + + private DoiCheck() { + } + + public static boolean isDoi(final String value) { + + Matcher m = PATTERN.matcher(purgeDoiValue(value)); + return m.matches(); + } + + + public static String purgeDoiValue(final String query) { + String value = query.replaceAll(",", ""); + for (final String prefix : DOI_PREFIXES) { + value = value.replaceAll(prefix, ""); + } + return value.trim(); + } + +} diff --git a/dspace-api/src/test/data/dspaceFolder/config/spring/api/external-services.xml b/dspace-api/src/test/data/dspaceFolder/config/spring/api/external-services.xml index ac163d3581..2157556d46 100644 --- a/dspace-api/src/test/data/dspaceFolder/config/spring/api/external-services.xml +++ b/dspace-api/src/test/data/dspaceFolder/config/spring/api/external-services.xml @@ -6,6 +6,8 @@ + + diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/AbstractLiveImportIntegrationTest.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/AbstractLiveImportIntegrationTest.java new file mode 100644 index 0000000000..ca3195b344 --- /dev/null +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/AbstractLiveImportIntegrationTest.java @@ -0,0 +1,80 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.UnsupportedEncodingException; +import java.util.Collection; + +import org.apache.http.ProtocolVersion; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.entity.BasicHttpEntity; +import org.apache.tools.ant.filters.StringInputStream; +import org.dspace.app.rest.test.AbstractControllerIntegrationTest; +import org.dspace.importer.external.datamodel.ImportRecord; +import org.dspace.importer.external.metadatamapping.MetadatumDTO; + +/** + * @author Mykhaylo Boychuk (mykhaylo.boychuk at 4science.com) + */ +public class AbstractLiveImportIntegrationTest extends AbstractControllerIntegrationTest { + + protected boolean matchRecords(Collection recordsImported, Collection records2match) { + ImportRecord firstImported = recordsImported.iterator().next(); + ImportRecord secondImported = recordsImported.iterator().next(); + ImportRecord first2match = recordsImported.iterator().next(); + ImportRecord second2match = recordsImported.iterator().next(); + boolean checkFirstRecord = firstImported.getValueList().containsAll(first2match.getValueList()); + boolean checkSecondRecord = secondImported.getValueList().containsAll(second2match.getValueList()); + return checkFirstRecord && checkSecondRecord; + } + + protected MetadatumDTO createMetadatumDTO(String schema, String element, String qualifier, String value) { + MetadatumDTO metadatumDTO = new MetadatumDTO(); + metadatumDTO.setSchema(schema); + metadatumDTO.setElement(element); + metadatumDTO.setQualifier(qualifier); + metadatumDTO.setValue(value); + return metadatumDTO; + } + + protected CloseableHttpResponse mockResponse(String xmlExample, int statusCode, String reason) + throws UnsupportedEncodingException { + BasicHttpEntity basicHttpEntity = new BasicHttpEntity(); + basicHttpEntity.setChunked(true); + basicHttpEntity.setContent(new StringInputStream(xmlExample)); + + CloseableHttpResponse response = mock(CloseableHttpResponse.class); + when(response.getStatusLine()).thenReturn(statusLine(statusCode, reason)); + when(response.getEntity()).thenReturn(basicHttpEntity); + return response; + } + + protected StatusLine statusLine(int statusCode, String reason) { + return new StatusLine() { + @Override + public ProtocolVersion getProtocolVersion() { + return new ProtocolVersion("http", 1, 1); + } + + @Override + public int getStatusCode() { + return statusCode; + } + + @Override + public String getReasonPhrase() { + return reason; + } + }; + } + +} \ No newline at end of file