Merge remote-tracking branch 'upstream/main' into issue-1712_w2p-97080_facet-search-all-words-main

Merge remote-tracking branch 'upstream/main' into
issue-1712_w2p-97080_facet-search-all-words-main

dspace-api/src/main/java/org/dspace/importer/external/datacite/DataCiteFieldMapping.java

@@ -0,0 +1,38 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE and NOTICE files at the root of the source
+ * tree and available online at
+ *
+ * http://www.dspace.org/license/
+ */
+package org.dspace.importer.external.datacite;
+
+import java.util.Map;
+import javax.annotation.Resource;
+
+import org.dspace.importer.external.metadatamapping.AbstractMetadataFieldMapping;
+
+/**
+ * An implementation of {@link AbstractMetadataFieldMapping}
+ * Responsible for defining the mapping of the datacite metadatum fields on the DSpace metadatum fields
+ *
+ * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it)
+ * @author Florian Gantner (florian.gantner@uni-bamberg.de)
+ */
+public class DataCiteFieldMapping extends AbstractMetadataFieldMapping {
+
+    /**
+     * Defines which metadatum is mapped on which metadatum. Note that while the key must be unique it
+     * only matters here for postprocessing of the value. The mapped MetadatumContributor has full control over
+     * what metadatafield is generated.
+     *
+     * @param metadataFieldMap The map containing the link between retrieve metadata and metadata that will be set to
+     *                         the item.
+     */
+    @Override
+    @Resource(name = "dataciteMetadataFieldMap")
+    public void setMetadataFieldMap(Map metadataFieldMap) {
+        super.setMetadataFieldMap(metadataFieldMap);
+    }
+
+}

dspace-api/src/main/java/org/dspace/importer/external/datacite/DataCiteImportMetadataSourceServiceImpl.java

@@ -0,0 +1,168 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE and NOTICE files at the root of the source
+ * tree and available online at
+ *
+ * http://www.dspace.org/license/
+ */
+package org.dspace.importer.external.datacite;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import javax.el.MethodNotFoundException;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.dspace.content.Item;
+import org.dspace.importer.external.datamodel.ImportRecord;
+import org.dspace.importer.external.datamodel.Query;
+import org.dspace.importer.external.exception.MetadataSourceException;
+import org.dspace.importer.external.liveimportclient.service.LiveImportClient;
+import org.dspace.importer.external.service.AbstractImportMetadataSourceService;
+import org.dspace.importer.external.service.DoiCheck;
+import org.dspace.importer.external.service.components.QuerySource;
+import org.dspace.services.ConfigurationService;
+import org.springframework.beans.factory.annotation.Autowired;
+
+/**
+ * Implements a data source for querying Datacite
+ * Mainly copied from CrossRefImportMetadataSourceServiceImpl.
+ *
+ * optional Affiliation informations are not part of the API request.
+ * https://support.datacite.org/docs/can-i-see-more-detailed-affiliation-information-in-the-rest-api
+ *
+ * @author Pasquale Cavallo (pasquale.cavallo at 4science dot it)
+ * @author Florian Gantner (florian.gantner@uni-bamberg.de)
+ *
+ */
+public class DataCiteImportMetadataSourceServiceImpl
+        extends AbstractImportMetadataSourceService<String> implements QuerySource {
+    private final static Logger log = LogManager.getLogger();
+
+    @Autowired
+    private LiveImportClient liveImportClient;
+
+    @Autowired
+    private ConfigurationService configurationService;
+
+    @Override
+    public String getImportSource() {
+        return "datacite";
+    }
+
+    @Override
+    public void init() throws Exception {
+    }
+
+    @Override
+    public ImportRecord getRecord(String recordId) throws MetadataSourceException {
+        Collection<ImportRecord> records = getRecords(recordId, 0, 1);
+        if (records.size() == 0) {
+            return null;
+        }
+        return records.stream().findFirst().get();
+    }
+
+    @Override
+    public int getRecordsCount(String query) throws MetadataSourceException {
+        Collection<ImportRecord> records = getRecords(query, 0, -1);
+        return records == null ? 0 : records.size();
+    }
+
+    @Override
+    public int getRecordsCount(Query query) throws MetadataSourceException {
+        String id = getID(query.toString());
+        return getRecordsCount(StringUtils.isBlank(id) ? query.toString() : id);
+    }
+
+
+    @Override
+    public Collection<ImportRecord> getRecords(String query, int start, int count) throws MetadataSourceException {
+        List<ImportRecord> records = new ArrayList<>();
+        String id = getID(query);
+        Map<String, Map<String, String>> params = new HashMap<>();
+        Map<String, String> uriParameters = new HashMap<>();
+        params.put("uriParameters", uriParameters);
+        if (StringUtils.isBlank(id)) {
+            id = query;
+        }
+        uriParameters.put("query", id);
+        int timeoutMs = configurationService.getIntProperty("datacite.timeout", 180000);
+        String url = configurationService.getProperty("datacite.url", "https://api.datacite.org/dois/");
+        String responseString = liveImportClient.executeHttpGetRequest(timeoutMs, url, params);
+        JsonNode jsonNode = convertStringJsonToJsonNode(responseString);
+        if (jsonNode == null) {
+            log.warn("DataCite returned invalid JSON");
+            return records;
+        }
+        JsonNode dataNode = jsonNode.at("/data");
+        if (dataNode.isArray()) {
+            Iterator<JsonNode> iterator = dataNode.iterator();
+            while (iterator.hasNext()) {
+                JsonNode singleDoiNode = iterator.next();
+                String json = singleDoiNode.at("/attributes").toString();
+                records.add(transformSourceRecords(json));
+            }
+        } else {
+            String json = dataNode.at("/attributes").toString();
+            records.add(transformSourceRecords(json));
+        }
+
+        return records;
+    }
+
+    private JsonNode convertStringJsonToJsonNode(String json) {
+        try {
+            return new ObjectMapper().readTree(json);
+        } catch (JsonProcessingException e) {
+            log.error("Unable to process json response.", e);
+        }
+        return null;
+    }
+
+    @Override
+    public Collection<ImportRecord> getRecords(Query query) throws MetadataSourceException {
+        String id = getID(query.toString());
+        return getRecords(StringUtils.isBlank(id) ? query.toString() : id, 0, -1);
+    }
+
+    @Override
+    public ImportRecord getRecord(Query query) throws MetadataSourceException {
+        String id = getID(query.toString());
+        return getRecord(StringUtils.isBlank(id) ? query.toString() : id);
+    }
+
+    @Override
+    public Collection<ImportRecord> findMatchingRecords(Query query) throws MetadataSourceException {
+        String id = getID(query.toString());
+        return getRecords(StringUtils.isBlank(id) ? query.toString() : id, 0, -1);
+    }
+
+
+    @Override
+    public Collection<ImportRecord> findMatchingRecords(Item item) throws MetadataSourceException {
+        throw new MethodNotFoundException("This method is not implemented for DataCite");
+    }
+
+    public String getID(String query) {
+        if (DoiCheck.isDoi(query)) {
+            return query;
+        }
+        // Workaround for encoded slashes.
+        if (query.contains("%252F")) {
+            query = query.replace("%252F", "/");
+        }
+        if (DoiCheck.isDoi(query)) {
+            return query;
+        }
+        return StringUtils.EMPTY;
+    }
+}

dspace-api/src/main/java/org/dspace/storage/rdbms/DatabaseUtils.java

@@ -26,6 +26,7 @@ import java.util.regex.Pattern;
 import javax.sql.DataSource;
 
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.ArrayUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.Logger;
 import org.dspace.core.Context;
@@ -37,6 +38,7 @@ import org.dspace.workflow.factory.WorkflowServiceFactory;
 import org.flywaydb.core.Flyway;
 import org.flywaydb.core.api.FlywayException;
 import org.flywaydb.core.api.MigrationInfo;
+import org.flywaydb.core.api.MigrationVersion;
 import org.flywaydb.core.api.callback.Callback;
 import org.flywaydb.core.api.configuration.FluentConfiguration;
 import org.flywaydb.core.internal.info.MigrationInfoDumper;
@@ -93,7 +95,7 @@ public class DatabaseUtils {
         // Usage checks
         if (argv.length < 1) {
             System.out.println("\nDatabase action argument is missing.");
-            System.out.println("Valid actions: 'test', 'info', 'migrate', 'repair', 'validate', " +
+            System.out.println("Valid actions: 'test', 'info', 'migrate', 'repair', 'skip', 'validate', " +
                     "'update-sequences' or 'clean'");
             System.out.println("\nOr, type 'database help' for more information.\n");
             System.exit(1);
@@ -111,8 +113,10 @@ public class DatabaseUtils {
             // *before* any other Flyway commands can be run. This is a safety check.
             FlywayUpgradeUtils.upgradeFlywayTable(flyway, dataSource.getConnection());
 
+            // Determine action param passed to "./dspace database"
+            switch (argv[0].toLowerCase(Locale.ENGLISH)) {
                 // "test" = Test Database Connection
-            if (argv[0].equalsIgnoreCase("test")) {
+                case "test":
                     // Try to connect to the database
                     System.out.println("\nAttempting to connect to database");
                     try (Connection connection = dataSource.getConnection()) {
@@ -137,7 +141,10 @@ public class DatabaseUtils {
                         sqle.printStackTrace(System.err);
                         System.exit(1);
                     }
-            } else if (argv[0].equalsIgnoreCase("info") || argv[0].equalsIgnoreCase("status")) {
+                    break;
+                // "info" and "status" are identical and provide database info
+                case "info":
+                case "status":
                     try (Connection connection = dataSource.getConnection()) {
                         // Print basic Database info
                         printDBInfo(connection);
@@ -150,16 +157,18 @@ public class DatabaseUtils {
                         // See: http://flywaydb.org/documentation/faq.html#case-sensitive
                         if (!tableExists(connection, flyway.getConfiguration().getTable(), true)) {
                             System.out
-                            .println("\nNOTE: This database is NOT yet initialized for auto-migrations (via Flyway).");
+                                .println("\nNOTE: This database is NOT yet initialized for auto-migrations " +
+                                             "(via Flyway).");
                             // Determine which version of DSpace this looks like
                             String dbVersion = determineDBVersion(connection);
                             if (dbVersion != null) {
                                 System.out
                                     .println("\nYour database looks to be compatible with DSpace version " + dbVersion);
                                 System.out.println(
-                                "All upgrades *after* version " + dbVersion + " will be run during the next migration" +
+                                    "All upgrades *after* version " + dbVersion + " will be run during the next " +
-                                    ".");
+                                        "migration.");
-                            System.out.println("\nIf you'd like to upgrade now, simply run 'dspace database migrate'.");
+                                System.out.println("\nIf you'd like to upgrade now, simply run 'dspace database " +
+                                                       "migrate'.");
                             }
                         }
 
@@ -177,13 +186,15 @@ public class DatabaseUtils {
                         e.printStackTrace(System.err);
                         System.exit(1);
                     }
-            } else if (argv[0].equalsIgnoreCase("migrate")) {
+                    break;
+                // "migrate" = Run all pending database migrations
+                case "migrate":
                     try (Connection connection = dataSource.getConnection()) {
                         System.out.println("\nDatabase URL: " + connection.getMetaData().getURL());
 
                         // "migrate" allows for an OPTIONAL second argument (only one may be specified):
                         //    - "ignored" = Also run any previously "ignored" migrations during the migration
-                    //    - "force" = Even if no pending migrations exist, still run a migration to trigger callbacks.
+                        //    - "force" = Even if no pending migrations exist, still run migrate to trigger callbacks.
                         //    - [version] = ONLY run migrations up to a specific DSpace version (ONLY FOR TESTING)
                         if (argv.length == 2) {
                             if (argv[1].equalsIgnoreCase("ignored")) {
@@ -203,25 +214,25 @@ public class DatabaseUtils {
                                         new InputStreamReader(System.in, StandardCharsets.UTF_8));
 
                                 System.out.println(
-                                "You've specified to migrate your database ONLY to version " + migrationVersion + " " +
+                                    "You've specified to migrate your database ONLY to version " + migrationVersion +
                                         " ...");
                                 System.out.println(
-                                "\nWARNING: In this mode, we DISABLE all callbacks, which means that you will need " +
+                                    "\nWARNING: In this mode, we DISABLE all callbacks, which means that you will " +
-                                    "to manually update registries and manually run a reindex. This is because you " +
+                                        "need to manually update registries and manually run a reindex. This is " +
-                                    "are attempting to use an OLD version (" + migrationVersion + ") Database with " +
+                                        "because you are attempting to use an OLD version (" + migrationVersion + ") " +
-                                    "a newer DSpace API. NEVER do this in a PRODUCTION scenario. The resulting " +
+                                        "Database with a newer DSpace API. NEVER do this in a PRODUCTION scenario. " +
-                                    "database is only useful for migration testing.\n");
+                                        "The resulting database is only useful for migration testing.\n");
 
                                 System.out.print(
-                                "Are you SURE you only want to migrate your database to version " + migrationVersion
+                                    "Are you SURE you only want to migrate your database to version " +
-                                    + "? [y/n]: ");
+                                        migrationVersion + "? [y/n]: ");
                                 String choiceString = input.readLine();
                                 input.close();
 
                                 if (choiceString.equalsIgnoreCase("y")) {
                                     System.out.println(
-                                    "Migrating database ONLY to version " + migrationVersion + " ... (Check logs for " +
+                                        "Migrating database ONLY to version " + migrationVersion + " ... " +
-                                        "details)");
+                                            "(Check logs for details)");
                                     // Update the database, to the version specified.
                                     updateDatabase(dataSource, connection, migrationVersion, false);
                                 } else {
@@ -229,7 +240,8 @@ public class DatabaseUtils {
                                 }
                             }
                         } else {
-                        System.out.println("Migrating database to latest version... (Check dspace logs for details)");
+                            System.out.println("Migrating database to latest version... " +
+                                                   "(Check dspace logs for details)");
                             updateDatabase(dataSource, connection);
                         }
                         System.out.println("Done.");
@@ -239,9 +251,9 @@ public class DatabaseUtils {
                         e.printStackTrace(System.err);
                         System.exit(1);
                     }
-            } else if (argv[0].equalsIgnoreCase("repair")) {
+                    break;
                 // "repair" = Run Flyway repair script
-
+                case "repair":
                     try (Connection connection = dataSource.getConnection();) {
                         System.out.println("\nDatabase URL: " + connection.getMetaData().getURL());
                         System.out.println(
@@ -255,32 +267,71 @@ public class DatabaseUtils {
                         e.printStackTrace(System.err);
                         System.exit(1);
                     }
-            } else if (argv[0].equalsIgnoreCase("validate")) {
+                    break;
-                // "validate" = Run Flyway validation to check for database errors/issues
+                // "skip" = Skip a specific Flyway migration (by telling Flyway it succeeded)
+                case "skip":
+                    try {
+                        // "skip" requires a migration version to skip. Only that exact version will be skipped.
+                        if (argv.length == 2) {
+                            String migrationVersion = argv[1];
 
+                            BufferedReader input = new BufferedReader(
+                                new InputStreamReader(System.in, StandardCharsets.UTF_8));
+                            System.out.println(
+                                "You've specified to SKIP the migration with version='" + migrationVersion + "' " +
+                                    "...");
+                            System.out.print(
+                                "\nWARNING: You should only skip migrations which are no longer required or have " +
+                                    "become obsolete. Skipping a REQUIRED migration may result in DSpace failing " +
+                                    "to startup or function properly. Are you sure you want to SKIP the " +
+                                    "migration with version '" + migrationVersion + "'? [y/n]: ");
+                            String choiceString = input.readLine();
+                            input.close();
+
+                            if (choiceString.equalsIgnoreCase("y")) {
+                                System.out.println(
+                                    "Attempting to skip migration with version " + migrationVersion + " " +
+                                        "... (Check logs for details)");
+                                skipMigration(dataSource, migrationVersion);
+                            }
+                        } else {
+                            System.out.println("The 'skip' command REQUIRES a version to be specified. " +
+                                                   "Only that single migration will be skipped. For the list " +
+                                                   "of migration versions use the 'info' command.");
+                        }
+                    } catch (IOException e) {
+                        System.err.println("Exception when attempting to skip migration:");
+                        e.printStackTrace(System.err);
+                        System.exit(1);
+                    }
+                    break;
+                // "validate" = Run Flyway validation to check for database errors/issues
+                case "validate":
                     try (Connection connection = dataSource.getConnection();) {
                         System.out.println("\nDatabase URL: " + connection.getMetaData().getURL());
                         System.out
-                        .println("Attempting to validate database status (and migration checksums) via FlywayDB...");
+                            .println("Attempting to validate database status (and migration checksums) via " +
+                                         "FlywayDB...");
                         flyway.validate();
-                    System.out.println("No errors thrown. Validation succeeded. (Check dspace logs for more details)");
+                        System.out.println("No errors thrown. Validation succeeded. (Check dspace logs for more " +
+                                               "details)");
                         System.exit(0);
                     } catch (SQLException | FlywayException e) {
                         System.err.println("Validation exception:");
                         e.printStackTrace(System.err);
                         System.exit(1);
                     }
-            } else if (argv[0].equalsIgnoreCase("clean")) {
+                    break;
                 // "clean" = Run Flyway clean script
-
+                case "clean":
                     // If clean is disabled, return immediately
                     if (flyway.getConfiguration().isCleanDisabled()) {
                         System.out.println(
-                        "\nWARNING: 'clean' command is currently disabled, as it is dangerous to run in Production " +
+                            "\nWARNING: 'clean' command is currently disabled, as it is dangerous to run in " +
-                            "scenarios!");
+                                "Production scenarios!");
                         System.out.println(
-                        "\nIn order to run a 'clean' you first must enable it in your DSpace config by specifying 'db" +
+                            "\nIn order to run a 'clean' you first must enable it in your DSpace config by " +
-                            ".cleanDisabled=false'.\n");
+                                "specifying 'db.cleanDisabled=false'.\n");
                         System.exit(1);
                     }
 
@@ -295,33 +346,35 @@ public class DatabaseUtils {
                                 String username = connection.getMetaData().getUserName();
                                 // Exit immediately, providing a descriptive error message
                                 System.out.println(
-                                "\nERROR: The database user '" + username + "' does not have sufficient privileges to" +
+                                    "\nERROR: The database user '" + username + "' does not have sufficient " +
-                                    " run a 'database clean' (via Flyway).");
+                                        "privileges to run a 'database clean' (via Flyway).");
                                 System.out.println(
                                     "\nIn order to run a 'clean', the database user MUST have 'superuser' privileges");
                                 System.out.println(
-                                "OR the '" + PostgresUtils.PGCRYPTO + "' extension must be installed in a separate " +
+                                    "OR the '" + PostgresUtils.PGCRYPTO + "' extension must be installed in a " +
-                                    "schema (see documentation).");
+                                        "separate schema (see documentation).");
                                 System.out.println(
-                                "\nOptionally, you could also manually remove the '" + PostgresUtils.PGCRYPTO + "' " +
+                                    "\nOptionally, you could also manually remove the '" + PostgresUtils.PGCRYPTO +
-                                    "extension first (DROP EXTENSION " + PostgresUtils.PGCRYPTO + " CASCADE;), then " +
+                                        "' extension first (DROP EXTENSION " + PostgresUtils.PGCRYPTO +
-                                    "rerun the 'clean'");
+                                        " CASCADE;), then rerun the 'clean'");
                                 System.exit(1);
                             }
                         }
 
-                    BufferedReader input = new BufferedReader(new InputStreamReader(System.in, StandardCharsets.UTF_8));
+                        BufferedReader input = new BufferedReader(new InputStreamReader(System.in,
+                                                                                        StandardCharsets.UTF_8));
 
                         System.out.println("\nDatabase URL: " + connection.getMetaData().getURL());
                         System.out
                             .println("\nWARNING: ALL DATA AND TABLES IN YOUR DATABASE WILL BE PERMANENTLY DELETED.\n");
-                    System.out.println("There is NO turning back from this action. Backup your DB before continuing.");
+                        System.out.println("There is NO turning back from this action. Backup your DB before " +
+                                               "continuing.");
                         if (dbType.equals(DBMS_ORACLE)) {
                             System.out.println("\nORACLE WARNING: your RECYCLEBIN will also be PURGED.\n");
                         } else if (dbType.equals(DBMS_POSTGRES)) {
                             System.out.println(
-                            "\nPOSTGRES WARNING: the '" + PostgresUtils.PGCRYPTO + "' extension will be dropped if it" +
+                                "\nPOSTGRES WARNING: the '" + PostgresUtils.PGCRYPTO + "' extension will be dropped " +
-                                " is in the same schema as the DSpace database.\n");
+                                "if it is in the same schema as the DSpace database.\n");
                         }
                         System.out.print("Do you want to PERMANENTLY DELETE everything from your database? [y/n]: ");
                         String choiceString = input.readLine();
@@ -340,14 +393,16 @@ public class DatabaseUtils {
                         e.printStackTrace(System.err);
                         System.exit(1);
                     }
-            } else if (argv[0].equalsIgnoreCase("update-sequences")) {
+                    break;
+                // "update-sequences" = Run DSpace's "update-sequences.sql" script
+                case "update-sequences":
                     try (Connection connection = dataSource.getConnection()) {
                         String dbType = getDbType(connection);
                         String sqlfile = "org/dspace/storage/rdbms/sqlmigration/" + dbType +
                                 "/update-sequences.sql";
                         InputStream sqlstream = DatabaseUtils.class.getClassLoader().getResourceAsStream(sqlfile);
                         if (sqlstream != null) {
-                        String s = IOUtils.toString(sqlstream, "UTF-8");
+                            String s = IOUtils.toString(sqlstream, StandardCharsets.UTF_8);
                             if (!s.isEmpty()) {
                                 System.out.println("Running " + sqlfile);
                                 connection.createStatement().execute(s);
@@ -359,10 +414,12 @@ public class DatabaseUtils {
                             System.err.println(sqlfile + " not found");
                         }
                     }
-            } else {
+                    break;
+                // default = show help information
+                default:
                     System.out.println("\nUsage: database [action]");
-                System.out.println("Valid actions: 'test', 'info', 'migrate', 'repair', " +
+                    System.out.println("Valid actions: 'test', 'info', 'migrate', 'repair', 'skip', " +
-                    "'update-sequences' or 'clean'");
+                        "'validate', 'update-sequences' or 'clean'");
                     System.out.println(
                         " - test             = Performs a test connection to database to " +
                         "validate connection settings");
@@ -374,6 +431,9 @@ public class DatabaseUtils {
                     System.out.println(
                         " - repair           = Attempt to repair any previously failed database " +
                         "migrations or checksum mismatches (via Flyway repair)");
+                    System.out.println(
+                        " - skip [version]   = Skip a single, pending or ignored migration, " +
+                        "ensuring it never runs.");
                     System.out.println(
                         " - validate         = Validate current database's migration status (via Flyway validate), " +
                         "validating all migration checksums.");
@@ -385,6 +445,7 @@ public class DatabaseUtils {
                         "Requires 'db.cleanDisabled=false' setting in config.");
                     System.out.println("");
                     System.exit(0);
+                    break;
             }
 
         } catch (Exception e) {
@@ -786,6 +847,89 @@ public class DatabaseUtils {
         }
     }
 
+    /**
+     * Skips the given migration by marking it as "successful" in the Flyway table. This ensures
+     * the given migration will never be run again.
+     * <P>
+     * WARNING: Skipping a required migration can result in unexpected errors. Make sure the migration is
+     * not required (or obsolete) before skipping it.
+     * @param dataSource current DataSource
+     * @param skipVersion version of migration to skip
+     * @throws SQLException if error occurs
+     */
+    private static synchronized void skipMigration(DataSource dataSource,
+                                                   String skipVersion) throws SQLException {
+        if (null == dataSource) {
+            throw new SQLException("The datasource is a null reference -- cannot continue.");
+        }
+
+        try (Connection connection = dataSource.getConnection()) {
+            // Setup Flyway API against our database
+            FluentConfiguration flywayConfiguration = setupFlyway(dataSource);
+
+            // In order to allow for skipping "Ignored" migrations, we MUST set "outOfOrder=true".
+            // (Otherwise Ignored migrations never appear in the pending list)
+            flywayConfiguration.outOfOrder(true);
+
+            // Initialized Flyway object based on this configuration
+            Flyway flyway = flywayConfiguration.load();
+
+            // Find the migration we are skipping in the list of pending migrations
+            boolean foundMigration = false;
+            for (MigrationInfo migration : flyway.info().pending()) {
+                // If this migration matches our "skipVersion"
+                if (migration.getVersion().equals(MigrationVersion.fromVersion(skipVersion))) {
+                    foundMigration = true;
+                    System.out.println("Found migration matching version='" + skipVersion + "'. " +
+                                           "Changing state to 'Success' in order to skip it.");
+
+                    PreparedStatement statement = null;
+                    try {
+                        // Create SQL Insert which will log this migration as having already been run.
+                        String INSERT_SQL = "INSERT INTO " + FLYWAY_TABLE  + " " +
+                            "(" +
+                              "installed_rank, version, description, type, script, " +
+                              "checksum, installed_by, execution_time, success" +
+                            ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)";
+                        statement = connection.prepareStatement(INSERT_SQL);
+                        // installed_rank
+                        statement.setInt(1, getNextFlywayInstalledRank(flyway));
+                        // version
+                        statement.setString(2, migration.getVersion().getVersion());
+                        // description
+                        statement.setString(3, migration.getDescription());
+                        // type
+                        statement.setString(4, migration.getType().toString());
+                        // script
+                        statement.setString(5, migration.getScript());
+                        // checksum
+                        statement.setInt(6, migration.getChecksum());
+                        // installed_by
+                        statement.setString(7, getDBUserName(connection));
+                        // execution_time is set to zero as we didn't really execute it
+                        statement.setInt(8, 0);
+                        // success=true tells Flyway this migration no longer needs to be run.
+                        statement.setBoolean(9, true);
+
+                        // Run the INSERT
+                        statement.executeUpdate();
+                    } finally {
+                        if (statement != null && !statement.isClosed()) {
+                            statement.close();
+                        }
+                    }
+                }
+            }
+            if (!foundMigration) {
+                System.err.println("Could not find migration to skip! " +
+                                       "No 'Pending' or 'Ignored' migrations match version='" + skipVersion + "'");
+            }
+        } catch (FlywayException fe) {
+            // If any FlywayException (Runtime) is thrown, change it to a SQLException
+            throw new SQLException("Flyway error occurred", fe);
+        }
+    }
+
     /**
      * Clean the existing database, permanently removing all data and tables
      * <P>
@@ -1192,6 +1336,34 @@ public class DatabaseUtils {
         return schema;
     }
 
+    /**
+     * Get the Database User Name in use by this Connection.
+     *
+     * @param connection Current Database Connection
+     * @return User name as a string, or "null" if cannot be determined or unspecified
+     * @throws SQLException An exception that provides information on a database access error or other errors.
+     */
+    public static String getDBUserName(Connection connection)
+        throws SQLException {
+        String username = null;
+
+        // Try to get the schema from the DB connection itself.
+        // As long as the Database driver supports JDBC4.1, there should be a getSchema() method
+        // If this method is unimplemented or doesn't exist, it will throw an exception (likely an AbstractMethodError)
+        try {
+            username = connection.getMetaData().getUserName();
+        } catch (Exception | AbstractMethodError e) {
+            // ignore
+        }
+
+        // If we don't know our schema, let's try the schema in the DSpace configuration
+        if (StringUtils.isBlank(username)) {
+            username = canonicalize(connection, DSpaceServicesFactory.getInstance().getConfigurationService()
+                                                                     .getProperty("db.username"));
+        }
+        return username;
+    }
+
     /**
      * Return the canonical name for a database identifier based on whether this
      * database defaults to storing identifiers in uppercase or lowercase.
@@ -1443,4 +1615,22 @@ public class DatabaseUtils {
         }
         return null;
     }
+
+    /**
+     * Determine next valid "installed_rank" value from Flyway, based on the "installed_rank" of the
+     * last applied migration.
+     * @param flyway currently loaded Flyway
+     * @return next installed rank value
+     */
+    private static int getNextFlywayInstalledRank(Flyway flyway) throws FlywayException {
+        // Load all applied migrations
+        MigrationInfo[] appliedMigrations = flyway.info().applied();
+        // If no applied migrations, throw an error.
+        // This should never happen, but this would mean Flyway is not installed or initialized
+        if (ArrayUtils.isEmpty(appliedMigrations)) {
+            throw new FlywayException("Cannot determine next 'installed_rank' as no applied migrations exist");
+        }
+        // Find the last migration in the list, and increment its "installed_rank" by one.
+        return appliedMigrations[appliedMigrations.length - 1].getInstalledRank() + 1;
+    }
 }

dspace-api/src/main/resources/spring/spring-dspace-addon-import-services.xml

@@ -34,6 +34,14 @@
          </property>
     </bean>
 
+    <bean id="DataCiteImportService"
+          class="org.dspace.importer.external.datacite.DataCiteImportMetadataSourceServiceImpl" scope="singleton">
+        <property name="metadataFieldMapping" ref="DataCiteMetadataFieldMapping"/>
+    </bean>
+    <bean id="DataCiteMetadataFieldMapping"
+          class="org.dspace.importer.external.datacite.DataCiteFieldMapping">
+    </bean>
+
     <bean id="ArXivImportService"
           class="org.dspace.importer.external.arxiv.service.ArXivImportMetadataSourceServiceImpl" scope="singleton">
         <property name="metadataFieldMapping" ref="ArXivMetadataFieldMapping"/>

dspace-api/src/test/data/dspaceFolder/config/spring/api/external-services.xml

@@ -91,4 +91,16 @@
         </property>
     </bean>
 
+    <bean id="dataciteLiveImportDataProvider" class="org.dspace.external.provider.impl.LiveImportDataProvider">
+        <property name="metadataSource" ref="DataCiteImportService"/>
+        <property name="sourceIdentifier" value="datacite"/>
+        <property name="recordIdMetadata" value="dc.identifier.doi"/>
+        <property name="supportedEntityTypes">
+            <list>
+                <value>Publication</value>
+                <value>none</value>
+            </list>
+        </property>
+    </bean>
 </beans>
+

dspace-server-webapp/src/main/java/org/dspace/app/rest/utils/HttpHeadersInitializer.java

@@ -163,6 +163,8 @@ public class HttpHeadersInitializer {
 
         }
 
+        httpHeaders.put(HttpHeaders.ACCESS_CONTROL_EXPOSE_HEADERS,
+                        Collections.singletonList(HttpHeaders.ACCEPT_RANGES));
         httpHeaders.put(CONTENT_DISPOSITION, Collections.singletonList(String.format(CONTENT_DISPOSITION_FORMAT,
                                                                                      disposition,
                                                                                      encodeText(fileName))));

dspace-server-webapp/src/test/java/org/dspace/app/rest/DataCiteImportMetadataSourceServiceIT.java

@@ -0,0 +1,149 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE and NOTICE files at the root of the source
+ * tree and available online at
+ *
+ * http://www.dspace.org/license/
+ */
+package org.dspace.app.rest;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.when;
+
+import java.io.InputStream;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import javax.el.MethodNotFoundException;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.dspace.builder.CollectionBuilder;
+import org.dspace.builder.CommunityBuilder;
+import org.dspace.builder.ItemBuilder;
+import org.dspace.content.Item;
+import org.dspace.importer.external.datacite.DataCiteImportMetadataSourceServiceImpl;
+import org.dspace.importer.external.datamodel.ImportRecord;
+import org.dspace.importer.external.liveimportclient.service.LiveImportClientImpl;
+import org.dspace.importer.external.metadatamapping.MetadatumDTO;
+import org.junit.Test;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mockito;
+import org.springframework.beans.factory.annotation.Autowired;
+
+
+/**
+ * Integration tests for {@link DataCiteImportMetadataSourceServiceImpl}
+ * 
+ * @author Mykhaylo Boychuk (mykhaylo.boychuk at 4science.com)
+ */
+public class DataCiteImportMetadataSourceServiceIT extends AbstractLiveImportIntegrationTest {
+
+    @Autowired
+    private LiveImportClientImpl liveImportClientImpl;
+
+    @Autowired
+    private DataCiteImportMetadataSourceServiceImpl dataCiteServiceImpl;
+
+    @Test
+    public void dataCiteImportMetadataGetRecordsTest() throws Exception {
+        context.turnOffAuthorisationSystem();
+        CloseableHttpClient originalHttpClient = liveImportClientImpl.getHttpClient();
+        CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class);
+        try (InputStream dataCiteResp = getClass().getResourceAsStream("dataCite-test.json")) {
+            String dataCiteRespXmlResp = IOUtils.toString(dataCiteResp, Charset.defaultCharset());
+
+            liveImportClientImpl.setHttpClient(httpClient);
+            CloseableHttpResponse response = mockResponse(dataCiteRespXmlResp, 200, "OK");
+            when(httpClient.execute(ArgumentMatchers.any())).thenReturn(response);
+
+            context.restoreAuthSystemState();
+            ArrayList<ImportRecord> collection2match = getRecords();
+            Collection<ImportRecord> recordsImported = dataCiteServiceImpl.getRecords("10.48550/arxiv.2207.04779",
+                    0, -1);
+            assertEquals(1, recordsImported.size());
+            matchRecords(new ArrayList<>(recordsImported), collection2match);
+        } finally {
+            liveImportClientImpl.setHttpClient(originalHttpClient);
+        }
+    }
+
+    @Test
+    public void dataCiteImportMetadataGetRecordsCountTest() throws Exception {
+        context.turnOffAuthorisationSystem();
+        CloseableHttpClient originalHttpClient = liveImportClientImpl.getHttpClient();
+        CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class);
+        try (InputStream dataciteResp = getClass().getResourceAsStream("dataCite-test.json")) {
+            String dataciteTextResp = IOUtils.toString(dataciteResp, Charset.defaultCharset());
+
+            liveImportClientImpl.setHttpClient(httpClient);
+            CloseableHttpResponse response = mockResponse(dataciteTextResp, 200, "OK");
+            when(httpClient.execute(ArgumentMatchers.any())).thenReturn(response);
+
+            context.restoreAuthSystemState();
+            int tot = dataCiteServiceImpl.getRecordsCount("10.48550/arxiv.2207.04779");
+            assertEquals(1, tot);
+        } finally {
+            liveImportClientImpl.setHttpClient(originalHttpClient);
+        }
+    }
+
+    @Test(expected = MethodNotFoundException.class)
+    public void dataCiteImportMetadataFindMatchingRecordsTest() throws Exception {
+        context.turnOffAuthorisationSystem();
+        parentCommunity = CommunityBuilder.createCommunity(context)
+                                          .withName("Parent Community")
+                                          .build();
+
+        org.dspace.content.Collection col1 = CollectionBuilder.createCollection(context, parentCommunity)
+                                                              .withName("Collection 1")
+                                                              .build();
+
+        Item testItem = ItemBuilder.createItem(context, col1)
+                                   .withTitle("test item")
+                                   .withIssueDate("2021")
+                                   .build();
+
+        context.restoreAuthSystemState();
+        dataCiteServiceImpl.findMatchingRecords(testItem);
+    }
+
+    private ArrayList<ImportRecord> getRecords() {
+        ArrayList<ImportRecord> records = new ArrayList<>();
+        //define first record
+        List<MetadatumDTO> metadatums  = new ArrayList<>();
+        MetadatumDTO title = createMetadatumDTO("dc", "title", null,
+                "Mathematical Proof Between Generations");
+        MetadatumDTO doi = createMetadatumDTO("dc", "identifier", "doi", "10.48550/arxiv.2207.04779");
+        MetadatumDTO author1 = createMetadatumDTO("dc", "contributor", "author", "Bayer, Jonas");
+        MetadatumDTO author2 = createMetadatumDTO("dc", "contributor", "author", "Benzmüller, Christoph");
+        MetadatumDTO author3 = createMetadatumDTO("dc", "contributor", "author", "Buzzard, Kevin");
+        MetadatumDTO author4 = createMetadatumDTO("dc", "contributor", "author", "David, Marco");
+        MetadatumDTO author5 = createMetadatumDTO("dc", "contributor", "author", "Lamport, Leslie");
+        MetadatumDTO author6 = createMetadatumDTO("dc", "contributor", "author", "Matiyasevich, Yuri");
+        MetadatumDTO author7 = createMetadatumDTO("dc", "contributor", "author", "Paulson, Lawrence");
+        MetadatumDTO author8 = createMetadatumDTO("dc", "contributor", "author", "Schleicher, Dierk");
+        MetadatumDTO author9 = createMetadatumDTO("dc", "contributor", "author", "Stock, Benedikt");
+        MetadatumDTO author10 = createMetadatumDTO("dc", "contributor", "author", "Zelmanov, Efim");
+        metadatums.add(title);
+        metadatums.add(doi);
+        metadatums.add(author1);
+        metadatums.add(author2);
+        metadatums.add(author3);
+        metadatums.add(author4);
+        metadatums.add(author5);
+        metadatums.add(author6);
+        metadatums.add(author7);
+        metadatums.add(author8);
+        metadatums.add(author9);
+        metadatums.add(author10);
+
+        ImportRecord firstRecord = new ImportRecord(metadatums);
+
+        records.add(firstRecord);
+        return records;
+    }
+
+}

dspace-server-webapp/src/test/java/org/dspace/app/rest/ExternalSourcesRestControllerIT.java

@@ -53,7 +53,7 @@ public class ExternalSourcesRestControllerIT extends AbstractControllerIntegrati
                                 ExternalSourceMatcher.matchExternalSource(
                                         "openAIREFunding", "openAIREFunding", false)
                             )))
-                            .andExpect(jsonPath("$.page.totalElements", Matchers.is(9)));
+                            .andExpect(jsonPath("$.page.totalElements", Matchers.is(10)));
     }
 
     @Test

dspace/config/modules/external-providers.cfg

@@ -85,4 +85,10 @@ scopus.search-api.viewMode =
 wos.apiKey =
 wos.url = https://wos-api.clarivate.com/api/wos/id/
 wos.url.search = https://wos-api.clarivate.com/api/wos/?databaseId=WOS&lang=en&usrQuery=
-##################################################################
+#################################################################
+#------------------------- DataCite ----------------------------#
+#---------------------------------------------------------------#
+
+datacite.url = https://api.datacite.org/dois/
+datacite.timeout = 180000
+#################################################################

dspace/config/registries/bitstream-formats.xml

@@ -187,6 +187,7 @@
     <internal>false</internal>
     <extension>jpeg</extension>
     <extension>jpg</extension>
+    <extension>jfif</extension>
   </bitstream-type>
   
   <bitstream-type>

dspace/config/spring/api/datacite-integration.xml

@@ -0,0 +1,62 @@
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns:context="http://www.springframework.org/schema/context"
+       xmlns:util="http://www.springframework.org/schema/util"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans
+           http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
+           http://www.springframework.org/schema/context
+           http://www.springframework.org/schema/context/spring-context-2.5.xsd http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd"
+       default-autowire-candidates="*Service,*DAO,javax.sql.DataSource">
+
+
+    <context:annotation-config/>
+    <!-- allows us to use spring annotations in beans -->
+
+    <util:map id="dataciteMetadataFieldMap" key-type="org.dspace.importer.external.metadatamapping.MetadataFieldConfig"
+              value-type="org.dspace.importer.external.metadatamapping.contributor.MetadataContributor">
+        <description>Defines which metadatum is mapped on which metadatum. Note that while the key must be unique it
+            only matters here for postprocessing of the value. The mapped MetadatumContributor has full control over
+            what metadatafield is generated.
+        </description>
+        <entry key-ref="datacite.title" value-ref="dataciteTitleContrib"/>
+        <entry key-ref="datacite.id" value-ref="dataciteIDContrib"/>
+        <entry key-ref="datacite.author" value-ref="dataciteAuthorContrib"/>
+        <!-- TODO: Further Mappings here! querys are applied among the data.attributes object containing the informations -->
+    </util:map>
+
+    <bean id="dataciteAuthorContrib" class="org.dspace.importer.external.metadatamapping.contributor.SimpleJsonPathMetadataContributor">
+        <property name="metadataProcessor">
+            <bean class="org.dspace.importer.external.metadatamapping.contributor.ArrayElementAttributeProcessor">
+                <property name="pathToArray" value="/creators"/>
+                <property name="elementAttribute" value="/name"/>
+            </bean>
+        </property>
+        <property name="field" ref="datacite.author"/>
+    </bean>
+    <bean id="datacite.author" class="org.dspace.importer.external.metadatamapping.MetadataFieldConfig">
+        <constructor-arg value="dc.contributor.author"/>
+    </bean>
+
+    <bean id="dataciteTitleContrib" class="org.dspace.importer.external.metadatamapping.contributor.SimpleJsonPathMetadataContributor">
+        <property name="metadataProcessor">
+            <bean class="org.dspace.importer.external.metadatamapping.contributor.ArrayElementAttributeProcessor">
+                <property name="pathToArray" value="/titles"/>
+                <property name="elementAttribute" value="/title"/>
+            </bean>
+        </property>
+        <property name="field" ref="datacite.title"/>
+    </bean>
+    <bean id="datacite.title" class="org.dspace.importer.external.metadatamapping.MetadataFieldConfig">
+        <constructor-arg value="dc.title"/>
+    </bean>
+
+    <!-- must be present to be imported, since it's used as the recordId-->
+    <bean id="dataciteIDContrib" class="org.dspace.importer.external.metadatamapping.contributor.SimpleJsonPathMetadataContributor">
+        <property name="field" ref="datacite.id"/>
+        <property name="query" value="/doi"/>
+    </bean>
+    <bean id="datacite.id" class="org.dspace.importer.external.metadatamapping.MetadataFieldConfig">
+        <constructor-arg value="dc.identifier.doi"/>
+    </bean>
+
+</beans>

dspace/config/spring/api/discovery.xml

@@ -247,6 +247,18 @@
                             <property name="field" value="dc.title"/>
                             <property name="snippets" value="5"/>
                         </bean>
+                        <bean class="org.dspace.discovery.configuration.DiscoveryHitHighlightFieldConfiguration">
+                            <property name="field" value="organization.legalName"/>
+                            <property name="snippets" value="5"/>
+                        </bean>
+                        <bean class="org.dspace.discovery.configuration.DiscoveryHitHighlightFieldConfiguration">
+                            <property name="field" value="person.givenName"/>
+                            <property name="snippets" value="5"/>
+                        </bean>
+                        <bean class="org.dspace.discovery.configuration.DiscoveryHitHighlightFieldConfiguration">
+                            <property name="field" value="person.familyName"/>
+                            <property name="snippets" value="5"/>
+                        </bean>
                         <!-- By default, full text snippets are disabled, as snippets of embargoed/restricted bitstreams
                              may appear in search results when the Item is public. See DS-3498
                         <bean class="org.dspace.discovery.configuration.DiscoveryHitHighlightFieldConfiguration">

dspace/config/spring/api/external-services.xml

@@ -212,4 +212,15 @@
         </property>
     </bean>
 
+    <bean id="dataciteLiveImportDataProvider" class="org.dspace.external.provider.impl.LiveImportDataProvider">
+        <property name="metadataSource" ref="DataCiteImportService"/>
+        <property name="sourceIdentifier" value="datacite"/>
+        <property name="recordIdMetadata" value="dc.identifier.doi"/>
+        <property name="supportedEntityTypes">
+            <list>
+                <value>Publication</value>
+                <value>none</value>
+            </list>
+        </property>
+    </bean>
 </beans>