hazza/DSpace
1
0
Fork 0
mirror of https://github.com/DSpace/DSpace.git synced 2025-10-07 01:54:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

116959 : Add Integration testing for JCloudBitStore service through BitstreamStorageManager to verify Bitstream state is properly set.

Browse Source
This commit is contained in:
Mark Diggory
2024-08-26 09:21:39 -04:00
parent fc9a21c8e9
commit ab32f4d6ef
9 changed files with 190 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
dspace-api/pom.xml
View File

@@ -841,15 +841,29 @@
</exclusions>
</dependency>
<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</dependency>
<dependency>
<groupId>org.apache.jclouds</groupId>
<artifactId>jclouds-core</artifactId>
<version>2.5.0</version>
<exclusions>
<exclusion>
<exclusion>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.xml.bind</groupId>
<artifactId>jaxb-impl</artifactId>

105
dspace-api/src/main/java/org/dspace/storage/bitstore/JCloudBitStoreService.java
View File

@@ -10,7 +10,6 @@ package org.dspace.storage.bitstore;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.sql.SQLException;
import java.util.Map;
import java.util.Properties;
@@ -19,6 +18,7 @@ import com.google.common.hash.HashCode;
import com.google.common.io.ByteSource;
import com.google.common.io.Files;
import com.google.common.net.MediaType;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -28,7 +28,6 @@ import org.dspace.core.Context;
import org.dspace.core.Utils;
import org.dspace.storage.bitstore.factory.StorageServiceFactory;
import org.dspace.storage.bitstore.service.BitstreamStorageService;
import org.dspace.utils.DSpace;
import org.jclouds.ContextBuilder;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.BlobStoreContext;
@@ -52,8 +51,17 @@ public class JCloudBitStoreService extends BaseBitStoreService {
private String providerOrApi;
private ContextBuilder builder;
private BlobStoreContext blobStoreContext;
/**
* container for all the assets
*/
private String container;
private String subFolder;
/**
* (Optional) subfolder within bucket where objects are stored
*/
private String subfolder = null;
private String identity;
private String credential;
private String endpoint;
@@ -89,8 +97,12 @@ public class JCloudBitStoreService extends BaseBitStoreService {
this.container = container;
}
public void setSubFolder(String subFolder) {
this.subFolder = subFolder;
public String getSubfolder() {
return subfolder;
}
public void setSubfolder(String subfolder) {
this.subfolder = subfolder;
}
public void setIdentity(String identity) {
@@ -132,10 +144,16 @@ public class JCloudBitStoreService extends BaseBitStoreService {
if (endpoint != null) {
this.builder = this.builder.endpoint(endpoint);
}
blobStoreContext = this.builder.overrides(properties)
.credentials(identity, credential).buildView(BlobStoreContext.class);
if (properties != null && !properties.isEmpty()) {
this.builder = this.builder.overrides(properties);
}
if (identity != null && credential != null) {
this.builder = this.builder.credentials(identity, credential);
}
blobStoreContext = this.builder.buildView(BlobStoreContext.class);
this.initialized = true;
} catch (Exception e) {
log.error(e.getMessage(),e);
this.initialized = false;
}
}
@@ -146,8 +164,7 @@ public class JCloudBitStoreService extends BaseBitStoreService {
if (counter == maxCounter) {
counter = 0;
blobStoreContext.close();
blobStoreContext = this.builder.overrides(properties)
.credentials(identity, credential).buildView(BlobStoreContext.class);
blobStoreContext = this.builder.buildView(BlobStoreContext.class);
}
}
@@ -188,8 +205,8 @@ public class JCloudBitStoreService extends BaseBitStoreService {
*/
public String getFullKey(String id) {
StringBuilder bufFilename = new StringBuilder();
if (StringUtils.isNotEmpty(this.subFolder)) {
bufFilename.append(this.subFolder);
if (StringUtils.isNotEmpty(this.subfolder)) {
bufFilename.append(this.subfolder);
appendSeparator(bufFilename);
}
@@ -200,7 +217,7 @@ public class JCloudBitStoreService extends BaseBitStoreService {
}
if (log.isDebugEnabled()) {
log.debug("S3 filepath for " + id + " is "
log.debug("Container filepath for " + id + " is "
+ bufFilename.toString());
}
@@ -253,7 +270,7 @@ public class JCloudBitStoreService extends BaseBitStoreService {
public void put(ByteSource byteSource, Bitstream bitstream) throws IOException {
final File file = getFile(bitstream);
String key = getFullKey(bitstream.getInternalId());
/* set type to sane default */
String type = MediaType.OCTET_STREAM.toString();
@@ -270,9 +287,9 @@ public class JCloudBitStoreService extends BaseBitStoreService {
blobStore.createContainerInLocation(null, container);
}
Blob blob = blobStore.blobBuilder(file.toString())
Blob blob = blobStore.blobBuilder(key)
.payload(byteSource)
.contentDisposition(file.toString())
.contentDisposition(key)
.contentLength(byteSource.size())
.contentType(type)
.build();
@@ -281,18 +298,42 @@ public class JCloudBitStoreService extends BaseBitStoreService {
blobStore.putBlob(container, blob, Builder.multipart());
}
/**
* Store a stream of bits.
*
* <p>
* If this method returns successfully, the bits have been stored.
* If an exception is thrown, the bits have not been stored.
* </p>
*
* @param in The stream of bits to store
* @throws java.io.IOException If a problem occurs while storing the bits
*/
@Override
public void put(Bitstream bitstream, InputStream in) throws IOException {
File tmp = File.createTempFile("jclouds", "cache");
String key = getFullKey(bitstream.getInternalId());
//Copy istream to temp file, and send the file, with some metadata
File scratchFile = File.createTempFile(bitstream.getInternalId(), "s3bs");
try {
// Inefficient caching strategy, however allows for use of JClouds store directly without CachingStore.
// Make sure there is sufficient storage in temp directory.
Files.asByteSink(tmp).writeFrom(in);
in.close();
put(Files.asByteSource(tmp), bitstream);
FileUtils.copyInputStreamToFile(in, scratchFile);
long contentLength = scratchFile.length();
// The ETag may or may not be and MD5 digest of the object data.
// Therefore, we precalculate before uploading
String localChecksum = org.dspace.curate.Utils.checksum(scratchFile, CSA);
put(Files.asByteSource(scratchFile), bitstream);
bitstream.setSizeBytes(contentLength);
bitstream.setChecksum(localChecksum);
bitstream.setChecksumAlgorithm(CSA);
} catch (Exception e) {
log.error("put(" + bitstream.getInternalId() + ", is)", e);
throw new IOException(e);
} finally {
if (!tmp.delete()) {
tmp.deleteOnExit();
if (!scratchFile.delete()) {
scratchFile.deleteOnExit();
}
}
}
@@ -347,25 +388,7 @@ public class JCloudBitStoreService extends BaseBitStoreService {
return new File(id);
}
/**
* Gets the URI of the content within the store.
*
* @param id the bitstream internal id.
* @return the URI, which is a relative path to the content.
*/
@SuppressWarnings("unused") // used by AVS2
public URI getStoredURI(String id) {
String tempID = getFullKey(id);
if (log.isDebugEnabled()) {
log.debug("Local URI for " + id + " is " + tempID);
}
return URI.create(id);
}
private String getContainer() {
if (container == null) {
container = new DSpace().getConfigurationService().getProperty("dspace.name");
}
return container;
}
}

10
dspace-api/src/test/data/dspaceFolder/config/local.cfg
View File

@@ -171,3 +171,13 @@ choices.plugin.dspace.object.owner = EPersonAuthority
choices.presentation.dspace.object.owner = suggest
authority.controlled.dspace.object.owner = true
### JCloudSettings
# Enabled the JCloudStore
assetstore.s3.generic.enabled = true
assetstore.s3.generic.useRelativePath = false
assetstore.s3.generic.subfolder = assetstore
assetstore.s3.generic.provider = filesystem
assetstore.s3.generic.container = assetstore-jclouds-container
assetstore.s3.generic.awsAccessKey =
assetstore.s3.generic.awsSecretKey =

70
dspace-api/src/test/data/dspaceFolder/config/modules/assetstore.cfg
View File

@@ -1,70 +0,0 @@
#---------------------------------------------------------------#
#-----------------STORAGE CONFIGURATIONS------------------------#
#---------------------------------------------------------------#
# Configuration properties used by the bitstore.xml config file #
# #
#---------------------------------------------------------------#
# assetstore.dir, look at DSPACE/config/spring/api/bitstore.xml for more options
assetstore.dir = ${dspace.dir}/assetstore
# Configures the primary store to be local or S3.
# This value will be used as `incoming` default store inside the `bitstore.xml`
# Possible values are:
# - 0: to use the `localStore`;
# - 1: to use the `s3Store`.
# If you want to add additional assetstores, they must be added to that bitstore.xml
# and new values should be provided as key-value pairs in the `stores` map of the
# `bitstore.xml` configuration.
assetstore.index.primary = 0
#---------------------------------------------------------------#
#-------------- Amazon S3 Specific Configurations --------------#
#---------------------------------------------------------------#
# The below configurations are only used if the primary storename
# is set to 's3Store' or the 's3Store' is configured as a secondary store
# in your bitstore.xml
# Enables or disables the store initialization during startup, without initialization the store won't work.
# if changed to true, a lazy initialization will be tried on next store usage, be careful an excecption could be thrown
assetstore.s3.enabled = false
# For using a relative path (xx/xx/xx/xxx...) set to true, default it false
# When true: it splits the path into subfolders, each of these
# are 2-chars (2-bytes) length, the last is the filename and could have
# at max 3-chars (3-bytes).
# When false: is used the absolute path using full filename.
assetstore.s3.useRelativePath = false
# S3 bucket name to store assets in. If unspecified, by default DSpace will
# create a bucket based on the hostname of `dspace.ui.url` setting.
assetstore.s3.bucketName =
# Subfolder to organize assets within the bucket, in case this bucket
# is shared. Optional, default is root level of bucket
assetstore.s3.subfolder =
# please don't use root credentials in production but rely on the aws credentials default
# discovery mechanism to configure them (ENV VAR, EC2 Iam role, etc.)
# The preferred approach for security reason is to use the IAM user credentials, but isn't always possible.
# More information about credentials here: https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html
# More information about IAM usage here: https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/java-dg-roles.html
assetstore.s3.awsAccessKey =
assetstore.s3.awsSecretKey =
# If the credentials are left empty,
# then this setting is ignored and the default AWS region will be used.
assetstore.s3.awsRegionName =
### JCloudSettings
# Enabled the JCloudStore
assetstore.s3.generic.enabled = false
assetstore.s3.generic.useRelativePath = true
assetstore.s3.generic.subfolder = assetstore
assetstore.s3.endpoint =
assetstore.s3.generic.provider = filesystem
assetstore.s3.generic.awsAccessKey =
assetstore.s3.generic.awsSecretKey =

16
dspace-api/src/test/data/dspaceFolder/config/bitstore.xml → dspace-api/src/test/data/dspaceFolder/config/spring/api/bitstore.xml
View File

@@ -45,23 +45,19 @@
<bean name="jcloudStore" class="org.dspace.storage.bitstore.JCloudBitStoreService" scope="singleton" lazy-init="true">
<property name="enabled" value="${assetstore.s3.generic.enabled:false}"/>
<property name="useRelativePath" value="${assetstore.s3.generic.useRelativePath:false}"/>
<property name="providerOrApi" value="${assetstore.s3.generic.provider}"/>
<property name="container" value="${assetstore.s3.generic.container}"/>
<property name="endpoint" value="${assetstore.s3.endpoint}"/>
<property name="providerOrApi" value="aws-s3"/>
<property name="identity" value="${assetstore.s3.generic.awsAccessKey}"/>
<property name="credentials" value="${assetstore.s3.generic.awsSecretKey}"/>
<property name="maxCounter" value="${assetstore.s3.maxCounter:100}"/>
<property name="overrides">
<props>
<prop key="#{propertyBaseDir}">./local/filesystemstorage</prop>
<prop key="#{propertyBaseDir}">target/testing/dspace</prop>
</props>
</property>
<!-- Subfolder to organize assets within the bucket, in case this bucket is shared -->
<!--
Subfolder to organize assets within the bucket, in case this bucket is shared -->
<!-- Optional, default is root level of bucket -->
<property name="subFolder" value="${assetstore.s3.generic.subfolder}"/>
<property name="subfolder" value="${assetstore.s3.generic.subfolder}"/>
</bean>
<!-- <bean name="localStore2 ... -->

73
dspace-api/src/test/java/org/dspace/content/BitstreamJCloudBitstoreTest.java Normal file
View File

@@ -0,0 +1,73 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.content;
import static org.junit.Assert.assertTrue;
import org.dspace.content.factory.ContentServiceFactory;
import org.dspace.content.service.BitstreamFormatService;
import org.dspace.services.ConfigurationService;
import org.dspace.services.factory.DSpaceServicesFactory;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
/**
* Unit Tests for class Bitstream
*
* @author Mark Diggory
*/
public class BitstreamJCloudBitstoreTest extends BitstreamTest {
protected BitstreamFormatService bitstreamFormatService = ContentServiceFactory.getInstance()
.getBitstreamFormatService();
private final ConfigurationService configurationService
= DSpaceServicesFactory.getInstance().getConfigurationService();
/**
* This method will be run before every test as per @Before. It will
* initialize resources required for the tests.
*
* Other methods can be annotated with @Before here or in subclasses
* but no execution order is guaranteed
*/
@Before
@Override
public void init() {
configurationService.setProperty("assetstore.index.primary", "2");
super.init();
}
/**
* Test of getStoreNumber method, of class Bitstream.
*/
@Test
@Override
public void testGetStoreNumber() {
//stored in store 2 by default
assertTrue("testGetStoreNumber 2", bs.getStoreNumber() == 2);
}
/**
* This method will be run after every test as per @After. It will
* clean resources initialized by the @Before methods.
*
* Other methods can be annotated with @After here or in subclasses
* but no execution order is guaranteed
*/
@After
@Override
public void destroy() {
configurationService.setProperty("assetstore.index.primary", "0");
super.destroy();
}
}

2
dspace-api/src/test/java/org/dspace/content/BitstreamTest.java
View File

@@ -58,7 +58,7 @@ public class BitstreamTest extends AbstractDSpaceObjectTest {
/**
* BitStream instance for the tests
*/
private Bitstream bs;
protected Bitstream bs;
/**
* Spy of AuthorizeService to use for tests

10
dspace/config/spring/api/bitstore.xml
View File

@@ -38,11 +38,11 @@
<property name="subfolder" value="${assetstore.s3.subfolder}"/>
</bean>
<!-- Define JCloudStoreService bean -->
<!--
<util:constant static-field="org.jclouds.filesystem.reference.FilesystemConstants.PROPERTY_BASEDIR"
id="propertyBaseDir"/>
<!-- Define JCloudStoreService bean -->
<!--
<bean name="jcloudStore" class="org.dspace.storage.bitstore.JCloudBitStoreService" scope="singleton" lazy-init="true">
<property name="enabled" value="${assetstore.s3.generic.enabled:false}"/>
<property name="useRelativePath" value="${assetstore.s3.generic.useRelativePath:false}"/>
@@ -54,12 +54,6 @@
<property name="credentials" value="${assetstore.s3.generic.awsSecretKey}"/>
<property name="maxCounter" value="${assetstore.s3.maxCounter:100}"/>
<property name="overrides">
<props>
<prop key="#{propertyBaseDir}">./local/filesystemstorage</prop>
</props>
</property>
Subfolder to organize assets within the bucket, in case this bucket is shared
Optional, default is root level of bucket
<property name="subFolder" value="${assetstore.s3.generic.subfolder}"/>

20
pom.xml
View File

@@ -1359,7 +1359,7 @@
<groupId>net.handle</groupId>
<artifactId>handle</artifactId>
<version>9.3.0</version>
<exclusions>
<exclusions>
<!-- A later version is brought in by google-oauth-client -->
<exclusion>
<groupId>com.google.code.gson</groupId>
@@ -1738,6 +1738,24 @@
<groupId>com.google.oauth-client</groupId>
<artifactId>google-oauth-client</artifactId>
<version>1.33.3</version>
<exclusions>
<exclusion>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
<version>5.1.0</version> <!-- Update to the latest compatible version -->
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.8.8</version> <!-- Ensure this version is compatible with your code -->
</dependency>
<!-- Findbugs annotations -->