DSC-403 Porting to DSpace-CRIS 7 of the IAM support for S3

dspace-api/src/main/java/org/dspace/storage/bitstore/S3BitStoreService.java

@@ -14,11 +14,13 @@ import java.util.Map;
 
 import com.amazonaws.AmazonClientException;
 import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
 import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.regions.Region;
 import com.amazonaws.regions.Regions;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.amazonaws.services.s3.model.AmazonS3Exception;
 import com.amazonaws.services.s3.model.GetObjectRequest;
 import com.amazonaws.services.s3.model.ObjectMetadata;
@@ -88,13 +90,28 @@ public class S3BitStoreService implements BitStoreService {
      */
     @Override
     public void init() throws IOException {
-        if (StringUtils.isBlank(getAwsAccessKey()) || StringUtils.isBlank(getAwsSecretKey())) {
-            log.warn("Empty S3 access or secret");
+        if(StringUtils.isNotBlank(getAwsAccessKey()) && StringUtils.isNotBlank(getAwsSecretKey())) {
+            log.warn("Use local defined S3 credentials");
+            // region
+            Regions regions = Regions.DEFAULT_REGION;
+            if (StringUtils.isNotBlank(awsRegionName)) {
+                try {
+                    regions = Regions.fromName(awsRegionName);
+                } catch (IllegalArgumentException e) {
+                    log.warn("Invalid aws_region: " + awsRegionName);
+                }
             }
-
             // init client
             AWSCredentials awsCredentials = new BasicAWSCredentials(getAwsAccessKey(), getAwsSecretKey());
-        s3Service = new AmazonS3Client(awsCredentials);
+            s3Service = AmazonS3ClientBuilder.standard()
+                    .withCredentials(new AWSStaticCredentialsProvider(awsCredentials))
+                    .withRegion(regions)
+                    .build();
+            log.warn("S3 Region set to: " + regions.getName());
+        } else {
+            log.info("Using a IAM role or aws environment credentials");
+            s3Service = AmazonS3ClientBuilder.defaultClient();
+        }
 
         // bucket name
         if (StringUtils.isEmpty(bucketName)) {
@@ -114,18 +131,6 @@ public class S3BitStoreService implements BitStoreService {
             throw new IOException(e);
         }
 
-        // region
-        if (StringUtils.isNotBlank(awsRegionName)) {
-            try {
-                Regions regions = Regions.fromName(awsRegionName);
-                Region region = Region.getRegion(regions);
-                s3Service.setRegion(region);
-                log.info("S3 Region set to: " + region.getName());
-            } catch (IllegalArgumentException e) {
-                log.warn("Invalid aws_region: " + awsRegionName);
-            }
-        }
-
         log.info("AWS S3 Assetstore ready to go! bucket:" + bucketName);
     }
 

dspace/config/dspace.cfg

@@ -1589,3 +1589,4 @@ include = ${module_dir}/usage-statistics.cfg
 include = ${module_dir}/versioning.cfg
 include = ${module_dir}/workflow.cfg
 include = ${module_dir}/external-providers.cfg
+include = ${module_dir}/storage.cfg

dspace/config/local.cfg.EXAMPLE

@@ -244,3 +244,27 @@ db.schema = public
 
 # Maximum size of a multipart request (i.e. max total size of all files in one request)
 #spring.servlet.multipart.max-request-size = 512MB
+
+#---------------------------------------------------------------#
+#-----------------STORAGE CONFIGURATIONS------------------------#
+#---------------------------------------------------------------#
+# Use the localStore or the s3Store implementation
+assetstore.storename.0 = localStore
+
+## Assetstore S3 configuration, only used if the above configuration
+## is set to s3Store
+
+# S3 bucket name to store assets in, default would generate a bucket
+# based on the dspace host name
+assetstore.s3.bucketName =
+# Subfolder to organize assets within the bucket, in case this bucket
+# is shared. Optional, default is root level of bucket
+assetstore.s3.subfolder =
+
+# please do not use that in production but rely on the aws credentials
+# discovery mechanism to configure them (ENV VAR, EC2 Iam role, etc.)
+assetstore.s3.awsAccessKey =
+assetstore.s3.awsSecretKey =
+
+# to force the use of a specific region
+assetstore.s3.awsRegionName =

dspace/config/modules/storage.cfg

@@ -0,0 +1,28 @@
+#---------------------------------------------------------------#
+#-----------------STORAGE CONFIGURATIONS------------------------#
+#---------------------------------------------------------------#
+# Configuration properties used by the bitstore.xml config file #
+#                                                               #
+#---------------------------------------------------------------#
+
+# Use the localStore or the s3Store implementation
+assetstore.storename.0 = localStore
+
+## Assetstore S3 configuration, only used if the above configuration
+## is set to s3Store
+
+# S3 bucket name to store assets in, default would generate a bucket
+# based on the dspace host name
+assetstore.s3.bucketName =
+# Subfolder to organize assets within the bucket, in case this bucket
+# is shared. Optional, default is root level of bucket
+assetstore.s3.subfolder =
+
+# please do not use these in production but rely on the aws credentials
+# discovery mechanism to configure them (ENV VAR, EC2 Iam role, etc.)
+assetstore.s3.awsAccessKey =
+assetstore.s3.awsSecretKey =
+# to force the use of a specific region when credentials are provided
+# in this configuratin file. If credentials are left empty this prop
+# is ignored
+assetstore.s3.awsRegionName =

dspace/config/spring/api/bitstore.xml

@@ -7,7 +7,7 @@
         <property name="incoming" value="0"/>
         <property name="stores">
             <map>
-                <entry key="0" value-ref="localStore"/>
+                <entry key="0" value-ref="${assetstore.storename.0}"/>
                 <!--<entry key="1" value-ref="s3Store"/>-->
             </map>
         </property>
@@ -19,19 +19,19 @@
 
     <bean name="s3Store" class="org.dspace.storage.bitstore.S3BitStoreService" scope="singleton">
         <!-- AWS Security credentials, with policies for specified bucket -->
-        <property name="awsAccessKey" value=""/>
-        <property name="awsSecretKey" value=""/>
+        <property name="awsAccessKey" value="${assetstore.s3.awsAccessKey}"/>
+        <property name="awsSecretKey" value="${assetstore.s3.awsSecretKey}"/>
 
         <!-- S3 bucket name to store assets in. example: longsight-dspace-auk -->
-        <property name="bucketName" value=""/>
+        <property name="bucketName" value="${assetstore.s3.bucketName}"/>
 
         <!-- AWS S3 Region to use: {us-east-1, us-west-1, eu-west-1, eu-central-1, ap-southeast-1, ... } -->
         <!-- Optional, sdk default is us-east-1 -->
-        <property name="awsRegionName" value=""/>
+        <property name="awsRegionName" value="${assetstore.s3.awsRegionName}"/>
 
         <!-- Subfolder to organize assets within the bucket, in case this bucket is shared  -->
         <!-- Optional, default is root level of bucket -->
-        <property name="subfolder" value=""/>
+        <property name="subfolder" value="${assetstore.s3.subfolder}"/>
     </bean>
 
     <!-- <bean name="localStore2 ... -->