hazza/DSpace
1
0
Fork 0
mirror of https://github.com/DSpace/DSpace.git synced 2025-10-07 01:54:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature-jcloud-store-7.4' into feature-jcloud-store-7.6

Browse Source
This commit is contained in:
Nathan Buckingham
2024-09-05 14:10:34 -04:00
parent 5a43e6bcf1 fc9a21c8e9
commit 2385684e9c
10 changed files with 885 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
dspace-api/pom.xml
View File

@@ -641,11 +641,6 @@
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
@@ -679,6 +674,12 @@
<dependency>
<groupId>com.google.api-client</groupId>
<artifactId>google-api-client</artifactId>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava-jdk5</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.google.http-client</groupId>
@@ -833,7 +834,41 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.jclouds</groupId>
<artifactId>jclouds-core</artifactId>
<version>2.5.0</version>
<exclusions>
<exclusion>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.xml.bind</groupId>
<artifactId>jaxb-impl</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.jclouds</groupId>
<artifactId>jclouds-blobstore</artifactId>
<version>2.5.0</version>
</dependency>
<dependency>
<groupId>org.apache.jclouds.api</groupId>
<artifactId>filesystem</artifactId>
<version>2.5.0</version>
</dependency>
<dependency>
<groupId>org.apache.jclouds.provider</groupId>
<artifactId>aws-s3</artifactId>
<version>2.5.0</version>
</dependency>
<dependency>
<groupId>io.findify</groupId>
<artifactId>s3mock_2.13</artifactId>
@@ -850,7 +885,7 @@
</exclusion>
</exclusions>
</dependency>
</dependencies>
<dependencyManagement>

14
dspace-api/src/main/java/org/dspace/storage/bitstore/BaseBitStoreService.java
View File

@@ -75,7 +75,7 @@ public abstract class BaseBitStoreService implements BitStoreService {
* an attempt to make a path traversal attack, so ignore the path prefix. The
* internal-ID is supposed to be just a filename, so this will not affect normal
* operation.
*
*
* @param sInternalId
* @return Sanitized id
*/
@@ -88,7 +88,7 @@ public abstract class BaseBitStoreService implements BitStoreService {
/**
* Append separator to target {@code StringBuilder}
*
*
* @param path
*/
protected void appendSeparator(StringBuilder path) {
@@ -99,7 +99,7 @@ public abstract class BaseBitStoreService implements BitStoreService {
/**
* Utility that checks string ending with separator
*
*
* @param bufFilename
* @return
*/
@@ -111,7 +111,7 @@ public abstract class BaseBitStoreService implements BitStoreService {
* Splits internalId into several subpaths using {@code digitsPerLevel} that
* indicates the folder name length, and {@code direcoryLevels} that indicates
* the maximum number of subfolders.
*
*
* @param internalId bitStream identifier
* @param path
*/
@@ -127,7 +127,7 @@ public abstract class BaseBitStoreService implements BitStoreService {
/**
* Extract substring if is in range, otherwise will truncate to length
*
*
* @param internalId
* @param startIndex
* @param endIndex
@@ -142,7 +142,7 @@ public abstract class BaseBitStoreService implements BitStoreService {
/**
* Checks if the {@code String} is longer than {@code endIndex}
*
*
* @param internalId
* @param endIndex
* @return
@@ -153,7 +153,7 @@ public abstract class BaseBitStoreService implements BitStoreService {
/**
* Retrieves a map of useful metadata about the File (size, checksum, modified)
*
*
* @param file The File to analyze
* @param attrs The list of requested metadata values
* @return Map of updated metadatas / attrs

50
dspace-api/src/main/java/org/dspace/storage/bitstore/BitstreamByteSource.java Normal file
View File

@@ -0,0 +1,50 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.storage.bitstore;
import java.io.IOException;
import java.io.InputStream;
import java.sql.SQLException;
import com.google.common.io.ByteSource;
import org.dspace.authorize.AuthorizeException;
import org.dspace.content.Bitstream;
import org.dspace.content.factory.ContentServiceFactory;
import org.dspace.content.service.BitstreamService;
import org.dspace.core.Context;
public class BitstreamByteSource extends ByteSource {
private static final BitstreamService bitstreamService = ContentServiceFactory.getInstance().getBitstreamService();
public Bitstream getBitstream() {
return bitstream;
}
private final Bitstream bitstream;
public BitstreamByteSource(Bitstream bitstream) {
this.bitstream = bitstream;
}
@Override
public InputStream openStream() throws IOException {
try {
return bitstreamService.retrieve(new Context(), bitstream);
} catch (SQLException | AuthorizeException e) {
throw new IOException(e.getMessage(), e);
}
}
@Override
public long size() throws IOException {
return bitstream.getSizeBytes();
}
}

371
dspace-api/src/main/java/org/dspace/storage/bitstore/JCloudBitStoreService.java Normal file
View File

@@ -0,0 +1,371 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.storage.bitstore;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.sql.SQLException;
import java.util.Map;
import java.util.Properties;
import com.google.common.hash.HashCode;
import com.google.common.io.ByteSource;
import com.google.common.io.Files;
import com.google.common.net.MediaType;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.dspace.content.Bitstream;
import org.dspace.content.BitstreamFormat;
import org.dspace.core.Context;
import org.dspace.core.Utils;
import org.dspace.storage.bitstore.factory.StorageServiceFactory;
import org.dspace.storage.bitstore.service.BitstreamStorageService;
import org.dspace.utils.DSpace;
import org.jclouds.ContextBuilder;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.BlobMetadata;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.options.PutOptions.Builder;
import org.jclouds.io.ContentMetadata;
import org.jclouds.javax.annotation.Nullable;
/**
* JCloudBitstream asset store service
*
* @author Mark Diggory, Nathan Buckingham
*/
public class JCloudBitStoreService extends BaseBitStoreService {
private static final Logger log = LogManager.getLogger(JCloudBitStoreService.class);
private Properties properties;
private String providerOrApi;
private ContextBuilder builder;
private BlobStoreContext blobStoreContext;
private String container;
private String subFolder;
private String identity;
private String credential;
private String endpoint;
private boolean useRelativePath;
private boolean enabled = false;
private int counter = 0;
private int maxCounter = 100;
private static final String CSA = "MD5";
public JCloudBitStoreService() {
}
public JCloudBitStoreService(String providerOrApi) {
this.providerOrApi = providerOrApi;
}
protected JCloudBitStoreService(BlobStoreContext blobStoreContext, String providerOrApi) {
this.blobStoreContext = blobStoreContext;
this.providerOrApi = providerOrApi;
}
public void setUseRelativePath(boolean useRelativePath) {
this.useRelativePath = useRelativePath;
}
public void setEnabled(boolean enabled) {
this.enabled = enabled;
}
public void setContainer(String container) {
this.container = container;
}
public void setSubFolder(String subFolder) {
this.subFolder = subFolder;
}
public void setIdentity(String identity) {
this.identity = identity;
}
public void setCredentials(@Nullable String credential) {
this.credential = credential;
}
public void setProviderOrApi(String providerOrApi) {
this.providerOrApi = providerOrApi;
}
public void setEndpoint(String endpoint) {
this.endpoint = endpoint;
}
public void setMaxCounter(int maxCounter) {
this.maxCounter = maxCounter;
}
public void setOverrides(Properties overrides) {
this.properties = overrides;
}
@Override
public boolean isEnabled() {
return this.enabled;
}
@Override
public void init() throws IOException {
if (this.isInitialized()) {
return;
}
try {
this.builder = ContextBuilder.newBuilder(providerOrApi);
if (endpoint != null) {
this.builder = this.builder.endpoint(endpoint);
}
blobStoreContext = this.builder.overrides(properties)
.credentials(identity, credential).buildView(BlobStoreContext.class);
this.initialized = true;
} catch (Exception e) {
this.initialized = false;
}
}
private synchronized void refreshContextIfNeeded() {
counter++;
// Destroying and recreating the connection between JClouds and CloudFiles
if (counter == maxCounter) {
counter = 0;
blobStoreContext.close();
blobStoreContext = this.builder.overrides(properties)
.credentials(identity, credential).buildView(BlobStoreContext.class);
}
}
@Override
public String generateId() {
return Utils.generateKey();
}
@Override
public InputStream get(final Bitstream bitstream) throws IOException {
final File file = getFile(bitstream);
return get(file);
}
private InputStream get(File file) throws IOException {
BlobStore blobStore = blobStoreContext.getBlobStore();
if (blobStore.blobExists(getContainer(), file.toString())) {
Blob blob = blobStore.getBlob(getContainer(), file.toString());
refreshContextIfNeeded();
return blob.getPayload().openStream();
}
throw new IOException("File not found: " + file);
}
@Override
public void remove(Bitstream bitstream) throws IOException {
File file = getFile(bitstream);
BlobStore blobStore = blobStoreContext.getBlobStore();
blobStore.removeBlob(getContainer(), file.toString());
deleteParents(file);
}
/**
* Utility Method: Prefix the key with a subfolder, if this instance assets are stored within subfolder
*
* @param id DSpace bitstream internal ID
* @return full key prefixed with a subfolder, if applicable
*/
public String getFullKey(String id) {
StringBuilder bufFilename = new StringBuilder();
if (StringUtils.isNotEmpty(this.subFolder)) {
bufFilename.append(this.subFolder);
appendSeparator(bufFilename);
}
if (this.useRelativePath) {
bufFilename.append(getRelativePath(id));
} else {
bufFilename.append(id);
}
if (log.isDebugEnabled()) {
log.debug("S3 filepath for " + id + " is "
+ bufFilename.toString());
}
return bufFilename.toString();
}
/**
* there are 2 cases:
* - conventional bitstream, conventional storage
* - registered bitstream, conventional storage
* conventional bitstream: dspace ingested, dspace random name/path
* registered bitstream: registered to dspace, any name/path
*
* @param sInternalId
* @return Computed Relative path
*/
private String getRelativePath(String sInternalId) {
BitstreamStorageService bitstreamStorageService = StorageServiceFactory.getInstance()
.getBitstreamStorageService();
String sIntermediatePath = StringUtils.EMPTY;
if (bitstreamStorageService.isRegisteredBitstream(sInternalId)) {
sInternalId = sInternalId.substring(2);
} else {
sInternalId = sanitizeIdentifier(sInternalId);
sIntermediatePath = getIntermediatePath(sInternalId);
}
return sIntermediatePath + sInternalId;
}
private void deleteParents(File file) {
if (file == null) {
return;
}
final BlobStore blobStore = blobStoreContext.getBlobStore();
for (int i = 0; i < directoryLevels; i++) {
final File directory = file.getParentFile();
final ListContainerOptions options = new ListContainerOptions();
options.inDirectory(directory.getPath());
long blobs = blobStore.countBlobs(getContainer(), options);
if (blobs != 0) {
break;
}
blobStore.deleteDirectory(getContainer(), directory.getPath());
file = directory;
}
}
public void put(ByteSource byteSource, Bitstream bitstream) throws IOException {
final File file = getFile(bitstream);
/* set type to sane default */
String type = MediaType.OCTET_STREAM.toString();
/* attempt to get type if the source is a Bitstream */
if (byteSource instanceof BitstreamByteSource) {
type = getMIMEType(((BitstreamByteSource) byteSource).getBitstream());
}
BlobStore blobStore = blobStoreContext.getBlobStore();
String container = getContainer();
if (!blobStore.containerExists(container)) {
blobStore.createContainerInLocation(null, container);
}
Blob blob = blobStore.blobBuilder(file.toString())
.payload(byteSource)
.contentDisposition(file.toString())
.contentLength(byteSource.size())
.contentType(type)
.build();
/* Utilize large file transfer to S3 via multipart post */
blobStore.putBlob(container, blob, Builder.multipart());
}
@Override
public void put(Bitstream bitstream, InputStream in) throws IOException {
File tmp = File.createTempFile("jclouds", "cache");
try {
// Inefficient caching strategy, however allows for use of JClouds store directly without CachingStore.
// Make sure there is sufficient storage in temp directory.
Files.asByteSink(tmp).writeFrom(in);
in.close();
put(Files.asByteSource(tmp), bitstream);
} finally {
if (!tmp.delete()) {
tmp.deleteOnExit();
}
}
}
public static String getMIMEType(final Bitstream bitstream) {
try {
BitstreamFormat format = bitstream.getFormat(new Context());
return format == null ? null : format.getMIMEType();
} catch (SQLException ignored) {
throw new RuntimeException(ignored);
}
}
@Override
@SuppressWarnings("unchecked")
public Map about(Bitstream bitstream, Map attrs) throws IOException {
File file = getFile(bitstream);
BlobStore blobStore = blobStoreContext.getBlobStore();
BlobMetadata blobMetadata = blobStore.blobMetadata(getContainer(), file.toString());
if (blobMetadata != null) {
ContentMetadata contentMetadata = blobMetadata.getContentMetadata();
if (contentMetadata != null) {
attrs.put("size_bytes", String.valueOf(contentMetadata.getContentLength()));
final HashCode hashCode = contentMetadata.getContentMD5AsHashCode();
if (hashCode != null) {
attrs.put("checksum", Utils.toHex(contentMetadata.getContentMD5AsHashCode().asBytes()));
attrs.put("checksum_algorithm", CSA);
}
attrs.put("modified", String.valueOf(blobMetadata.getLastModified().getTime()));
attrs.put("ContentDisposition", contentMetadata.getContentDisposition());
attrs.put("ContentEncoding", contentMetadata.getContentEncoding());
attrs.put("ContentLanguage", contentMetadata.getContentLanguage());
attrs.put("ContentType", contentMetadata.getContentType());
if (contentMetadata.getExpires() != null) {
attrs.put("Expires", contentMetadata.getExpires().getTime());
}
}
return attrs;
}
return null;
}
public File getFile(Bitstream bitstream) throws IOException {
String id = bitstream.getInternalId();
id = getFullKey(id);
if (log.isDebugEnabled()) {
log.debug("Local filename for " + bitstream.getInternalId() + " is " + id);
}
return new File(id);
}
/**
* Gets the URI of the content within the store.
*
* @param id the bitstream internal id.
* @return the URI, which is a relative path to the content.
*/
@SuppressWarnings("unused") // used by AVS2
public URI getStoredURI(String id) {
String tempID = getFullKey(id);
if (log.isDebugEnabled()) {
log.debug("Local URI for " + id + " is " + tempID);
}
return URI.create(id);
}
private String getContainer() {
if (container == null) {
container = new DSpace().getConfigurationService().getProperty("dspace.name");
}
return container;
}
}

69
dspace-api/src/test/data/dspaceFolder/config/bitstore.xml Normal file
View File

@@ -0,0 +1,69 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:util="http://www.springframework.org/schema/util"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd http://www.springframework.org/schema/util https://www.springframework.org/schema/util/spring-util.xsd"
default-lazy-init="true">
<bean name="org.dspace.storage.bitstore.BitstreamStorageService" class="org.dspace.storage.bitstore.BitstreamStorageServiceImpl">
<property name="incoming" value="${assetstore.index.primary}"/>
<property name="stores">
<map>
<entry key="0" value-ref="localStore"/>
<entry key="1" value-ref="s3Store"/>
<entry key="2" value-ref="jcloudStore"/>
</map>
</property>
</bean>
<bean name="localStore" class="org.dspace.storage.bitstore.DSBitStoreService" scope="singleton">
<property name="baseDir" value="${assetstore.dir}"/>
</bean>
<bean name="s3Store" class="org.dspace.storage.bitstore.S3BitStoreService" scope="singleton" lazy-init="true">
<property name="enabled" value="${assetstore.s3.enabled}"/>
<!-- AWS Security credentials, with policies for specified bucket -->
<property name="awsAccessKey" value="${assetstore.s3.awsAccessKey}"/>
<property name="awsSecretKey" value="${assetstore.s3.awsSecretKey}"/>
<property name="useRelativePath" value="${assetstore.s3.useRelativePath}"/>
<!-- S3 bucket name to store assets in. example: longsight-dspace-auk -->
<property name="bucketName" value="${assetstore.s3.bucketName}"/>
<!-- AWS S3 Region to use: {us-east-1, us-west-1, eu-west-1, eu-central-1, ap-southeast-1, ... } -->
<!-- Optional, sdk default is us-east-1 -->
<property name="awsRegionName" value="${assetstore.s3.awsRegionName}"/>
<!-- Subfolder to organize assets within the bucket, in case this bucket is shared -->
<!-- Optional, default is root level of bucket -->
<property name="subfolder" value="${assetstore.s3.subfolder}"/>
</bean>
<util:constant static-field="org.jclouds.filesystem.reference.FilesystemConstants.PROPERTY_BASEDIR"
id="propertyBaseDir"/>
<!-- Define JCloudStoreService bean -->
<bean name="jcloudStore" class="org.dspace.storage.bitstore.JCloudBitStoreService" scope="singleton" lazy-init="true">
<property name="enabled" value="${assetstore.s3.generic.enabled:false}"/>
<property name="useRelativePath" value="${assetstore.s3.generic.useRelativePath:false}"/>
<property name="container" value="${assetstore.s3.generic.container}"/>
<property name="endpoint" value="${assetstore.s3.endpoint}"/>
<property name="providerOrApi" value="aws-s3"/>
<property name="identity" value="${assetstore.s3.generic.awsAccessKey}"/>
<property name="credentials" value="${assetstore.s3.generic.awsSecretKey}"/>
<property name="maxCounter" value="${assetstore.s3.maxCounter:100}"/>
<property name="overrides">
<props>
<prop key="#{propertyBaseDir}">./local/filesystemstorage</prop>
</props>
</property>
<!-- Subfolder to organize assets within the bucket, in case this bucket is shared -->
<!-- Optional, default is root level of bucket -->
<property name="subFolder" value="${assetstore.s3.generic.subfolder}"/>
</bean>
<!-- <bean name="localStore2 ... -->
<!-- <bean name="s3Store2 ... -->
</beans>

70
dspace-api/src/test/data/dspaceFolder/config/modules/assetstore.cfg Normal file
View File

@@ -0,0 +1,70 @@
#---------------------------------------------------------------#
#-----------------STORAGE CONFIGURATIONS------------------------#
#---------------------------------------------------------------#
# Configuration properties used by the bitstore.xml config file #
# #
#---------------------------------------------------------------#
# assetstore.dir, look at DSPACE/config/spring/api/bitstore.xml for more options
assetstore.dir = ${dspace.dir}/assetstore
# Configures the primary store to be local or S3.
# This value will be used as `incoming` default store inside the `bitstore.xml`
# Possible values are:
# - 0: to use the `localStore`;
# - 1: to use the `s3Store`.
# If you want to add additional assetstores, they must be added to that bitstore.xml
# and new values should be provided as key-value pairs in the `stores` map of the
# `bitstore.xml` configuration.
assetstore.index.primary = 0
#---------------------------------------------------------------#
#-------------- Amazon S3 Specific Configurations --------------#
#---------------------------------------------------------------#
# The below configurations are only used if the primary storename
# is set to 's3Store' or the 's3Store' is configured as a secondary store
# in your bitstore.xml
# Enables or disables the store initialization during startup, without initialization the store won't work.
# if changed to true, a lazy initialization will be tried on next store usage, be careful an excecption could be thrown
assetstore.s3.enabled = false
# For using a relative path (xx/xx/xx/xxx...) set to true, default it false
# When true: it splits the path into subfolders, each of these
# are 2-chars (2-bytes) length, the last is the filename and could have
# at max 3-chars (3-bytes).
# When false: is used the absolute path using full filename.
assetstore.s3.useRelativePath = false
# S3 bucket name to store assets in. If unspecified, by default DSpace will
# create a bucket based on the hostname of `dspace.ui.url` setting.
assetstore.s3.bucketName =
# Subfolder to organize assets within the bucket, in case this bucket
# is shared. Optional, default is root level of bucket
assetstore.s3.subfolder =
# please don't use root credentials in production but rely on the aws credentials default
# discovery mechanism to configure them (ENV VAR, EC2 Iam role, etc.)
# The preferred approach for security reason is to use the IAM user credentials, but isn't always possible.
# More information about credentials here: https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html
# More information about IAM usage here: https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/java-dg-roles.html
assetstore.s3.awsAccessKey =
assetstore.s3.awsSecretKey =
# If the credentials are left empty,
# then this setting is ignored and the default AWS region will be used.
assetstore.s3.awsRegionName =
### JCloudSettings
# Enabled the JCloudStore
assetstore.s3.generic.enabled = false
assetstore.s3.generic.useRelativePath = true
assetstore.s3.generic.subfolder = assetstore
assetstore.s3.endpoint =
assetstore.s3.generic.provider = filesystem
assetstore.s3.generic.awsAccessKey =
assetstore.s3.generic.awsSecretKey =

222
dspace-api/src/test/java/org/dspace/storage/bitstore/JCloudBitStoreServiceTest.java Normal file
View File

@@ -0,0 +1,222 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.storage.bitstore;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import com.google.common.io.ByteSource;
import org.dspace.AbstractUnitTest;
import org.dspace.content.Bitstream;
import org.hamcrest.Matchers;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.BlobBuilder;
import org.jclouds.blobstore.domain.BlobBuilder.PayloadBlobBuilder;
import org.jclouds.io.Payload;
import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentMatchers;
import org.mockito.Mock;
import org.mockito.Mockito;
/**
* @author Nathan Buckingham
*
*/
public class JCloudBitStoreServiceTest extends AbstractUnitTest {
private JCloudBitStoreService jCloudBitStoreService;
@Mock
private BlobStoreContext blobStoreContext;
@Mock
private BlobStore blobStore;
@Mock
private Bitstream bitstream;
@Before
public void setUp() throws Exception {
this.jCloudBitStoreService = new JCloudBitStoreService(blobStoreContext, "filesystem");
}
@Test
public void getBitstreamTest() throws Exception {
Blob blob = Mockito.mock(Blob.class);
Payload payload = Mockito.mock(Payload.class);
InputStream inputStream = Mockito.mock(InputStream.class);
when(blob.getPayload()).thenReturn(payload);
when(payload.openStream()).thenReturn(inputStream);
when(blobStoreContext.getBlobStore()).thenReturn(blobStore);
when(blobStore.getBlob(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(blob);
when(blobStore.blobExists(ArgumentMatchers.any(), any())).thenReturn(true);
assertThat(this.jCloudBitStoreService.get(bitstream), Matchers.equalTo(inputStream));
}
@Test
public void removeBitstreamTest() throws Exception {
String bitStreamId = "BitStreamId";
when(bitstream.getInternalId()).thenReturn(bitStreamId);
when(blobStoreContext.getBlobStore()).thenReturn(blobStore);
try {
this.jCloudBitStoreService.remove(bitstream);
} catch (Exception e) {
// will fail due to trying to remove files
}
verify(this.blobStore, Mockito.times(1)).removeBlob(ArgumentMatchers.any(), ArgumentMatchers.any());
}
@Test
public void replaceBitStreamTest() throws Exception {
Blob blob = Mockito.mock(Blob.class);
File file = Mockito.mock(File.class);
BlobBuilder blobBuilder = Mockito.mock(BlobBuilder.class);
PayloadBlobBuilder payloadBlobBuilder = Mockito.mock(PayloadBlobBuilder.class);
when(blobStoreContext.getBlobStore()).thenReturn(blobStore);
when(blobStore.blobBuilder(ArgumentMatchers.any())).thenReturn(blobBuilder);
when(blobBuilder.payload(ArgumentMatchers.any(ByteSource.class))).thenReturn(payloadBlobBuilder);
when(payloadBlobBuilder.contentDisposition(ArgumentMatchers.any())).thenReturn(payloadBlobBuilder);
when(payloadBlobBuilder.contentLength(ArgumentMatchers.any(long.class))).thenReturn(payloadBlobBuilder);
when(payloadBlobBuilder.contentType(ArgumentMatchers.any(String.class))).thenReturn(payloadBlobBuilder);
when(payloadBlobBuilder.build()).thenReturn(blob);
ByteSource byteSource = Mockito.mock(ByteSource.class);
String mockedTag = "1a7771d5fdd7bfdfc84033c70b1ba555";
this.jCloudBitStoreService.put(byteSource, bitstream);
verify(blobStore, Mockito.times(1)).putBlob(ArgumentMatchers.any(),
ArgumentMatchers.any(), ArgumentMatchers.any());
}
@Test
public void givenBitStreamIdentifierLongerThanPossibleWhenIntermediatePathIsComputedThenIsSplittedAndTruncated() {
String path = "01234567890123456789";
String computedPath = this.jCloudBitStoreService.getIntermediatePath(path);
String expectedPath = "01" + File.separator + "23" + File.separator + "45" + File.separator;
assertThat(computedPath, equalTo(expectedPath));
}
@Test
public void givenBitStreamIdentifierShorterThanAFolderLengthWhenIntermediatePathIsComputedThenIsSingleFolder() {
String path = "0";
String computedPath = this.jCloudBitStoreService.getIntermediatePath(path);
String expectedPath = "0" + File.separator;
assertThat(computedPath, equalTo(expectedPath));
}
@Test
public void givenPartialBitStreamIdentifierWhenIntermediatePathIsComputedThenIsCompletlySplitted() {
String path = "01234";
String computedPath = this.jCloudBitStoreService.getIntermediatePath(path);
String expectedPath = "01" + File.separator + "23" + File.separator + "4" + File.separator;
assertThat(computedPath, equalTo(expectedPath));
}
@Test
public void givenMaxLengthBitStreamIdentifierWhenIntermediatePathIsComputedThenIsSplittedAllAsSubfolder() {
String path = "012345";
String computedPath = this.jCloudBitStoreService.getIntermediatePath(path);
String expectedPath = "01" + File.separator + "23" + File.separator + "45" + File.separator;
assertThat(computedPath, equalTo(expectedPath));
}
@Test
public void givenBitStreamIdentifierWhenIntermediatePathIsComputedThenNotEndingDoubleSlash() throws IOException {
StringBuilder path = new StringBuilder("01");
String computedPath = this.jCloudBitStoreService.getIntermediatePath(path.toString());
int slashes = computeSlashes(path.toString());
assertThat(computedPath, Matchers.endsWith(File.separator));
assertThat(computedPath.split(File.separator).length, Matchers.equalTo(slashes));
path.append("2");
computedPath = this.jCloudBitStoreService.getIntermediatePath(path.toString());
assertThat(computedPath, Matchers.not(Matchers.endsWith(File.separator + File.separator)));
path.append("3");
computedPath = this.jCloudBitStoreService.getIntermediatePath(path.toString());
assertThat(computedPath, Matchers.not(Matchers.endsWith(File.separator + File.separator)));
path.append("4");
computedPath = this.jCloudBitStoreService.getIntermediatePath(path.toString());
assertThat(computedPath, Matchers.not(Matchers.endsWith(File.separator + File.separator)));
path.append("56789");
computedPath = this.jCloudBitStoreService.getIntermediatePath(path.toString());
assertThat(computedPath, Matchers.not(Matchers.endsWith(File.separator + File.separator)));
}
@Test
public void givenBitStreamIdentidierWhenIntermediatePathIsComputedThenMustBeSplitted() throws IOException {
StringBuilder path = new StringBuilder("01");
String computedPath = this.jCloudBitStoreService.getIntermediatePath(path.toString());
int slashes = computeSlashes(path.toString());
assertThat(computedPath, Matchers.endsWith(File.separator));
assertThat(computedPath.split(File.separator).length, Matchers.equalTo(slashes));
path.append("2");
computedPath = this.jCloudBitStoreService.getIntermediatePath(path.toString());
slashes = computeSlashes(path.toString());
assertThat(computedPath, Matchers.endsWith(File.separator));
assertThat(computedPath.split(File.separator).length, Matchers.equalTo(slashes));
path.append("3");
computedPath = this.jCloudBitStoreService.getIntermediatePath(path.toString());
slashes = computeSlashes(path.toString());
assertThat(computedPath, Matchers.endsWith(File.separator));
assertThat(computedPath.split(File.separator).length, Matchers.equalTo(slashes));
path.append("4");
computedPath = this.jCloudBitStoreService.getIntermediatePath(path.toString());
slashes = computeSlashes(path.toString());
assertThat(computedPath, Matchers.endsWith(File.separator));
assertThat(computedPath.split(File.separator).length, Matchers.equalTo(slashes));
path.append("56789");
computedPath = this.jCloudBitStoreService.getIntermediatePath(path.toString());
slashes = computeSlashes(path.toString());
assertThat(computedPath, Matchers.endsWith(File.separator));
assertThat(computedPath.split(File.separator).length, Matchers.equalTo(slashes));
}
@Test
public void givenBitStreamIdentifierWithSlashesWhenSanitizedThenSlashesMustBeRemoved() {
String sInternalId = new StringBuilder("01")
.append(File.separator)
.append("22")
.append(File.separator)
.append("33")
.append(File.separator)
.append("4455")
.toString();
String computedPath = this.jCloudBitStoreService.sanitizeIdentifier(sInternalId);
assertThat(computedPath, Matchers.not(Matchers.startsWith(File.separator)));
assertThat(computedPath, Matchers.not(Matchers.endsWith(File.separator)));
assertThat(computedPath, Matchers.not(Matchers.containsString(File.separator)));
}
private int computeSlashes(String internalId) {
int minimum = internalId.length();
int slashesPerLevel = minimum / S3BitStoreService.digitsPerLevel;
int odd = Math.min(1, minimum % S3BitStoreService.digitsPerLevel);
int slashes = slashesPerLevel + odd;
return Math.min(slashes, S3BitStoreService.directoryLevels);
}
}

20
dspace/config/modules/assetstore.cfg
View File

@@ -12,10 +12,10 @@ assetstore.dir = ${dspace.dir}/assetstore
# This value will be used as `incoming` default store inside the `bitstore.xml`
# Possible values are:
# - 0: to use the `localStore`;
# - 1: to use the `s3Store`.
# - 1: to use the `s3Store`.
# If you want to add additional assetstores, they must be added to that bitstore.xml
# and new values should be provided as key-value pairs in the `stores` map of the
# `bitstore.xml` configuration.
# `bitstore.xml` configuration.
assetstore.index.primary = 0
#---------------------------------------------------------------#
@@ -33,7 +33,7 @@ assetstore.s3.enabled = false
# When true: it splits the path into subfolders, each of these
# are 2-chars (2-bytes) length, the last is the filename and could have
# at max 3-chars (3-bytes).
# When false: is used the absolute path using full filename.
# When false: is used the absolute path using full filename.
assetstore.s3.useRelativePath = false
# S3 bucket name to store assets in. If unspecified, by default DSpace will
@@ -54,4 +54,16 @@ assetstore.s3.awsSecretKey =
# If the credentials are left empty,
# then this setting is ignored and the default AWS region will be used.
assetstore.s3.awsRegionName =
assetstore.s3.awsRegionName =
### JCloudSettings
# Enabled the JCloudStore
assetstore.s3.generic.enabled = false
assetstore.s3.generic.useRelativePath = false
assetstore.s3.generic.subfolder = assetstore
assetstore.s3.endpoint =
assetstore.s3.generic.container =
assetstore.s3.maxCounter = 100
assetstore.s3.generic.provider = aws-s3

34
dspace/config/spring/api/bitstore.xml
View File

@@ -1,7 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd" default-lazy-init="true">
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:util="http://www.springframework.org/schema/util"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd http://www.springframework.org/schema/util https://www.springframework.org/schema/util/spring-util.xsd"
default-lazy-init="true">
<bean name="org.dspace.storage.bitstore.BitstreamStorageService" class="org.dspace.storage.bitstore.BitstreamStorageServiceImpl">
<property name="incoming" value="${assetstore.index.primary}"/>
@@ -9,6 +10,7 @@
<map>
<entry key="0" value-ref="localStore"/>
<entry key="1" value-ref="s3Store"/>
<!--<entry key="2" value-ref="jcloudStore"/>-->
</map>
</property>
</bean>
@@ -36,6 +38,34 @@
<property name="subfolder" value="${assetstore.s3.subfolder}"/>
</bean>
<util:constant static-field="org.jclouds.filesystem.reference.FilesystemConstants.PROPERTY_BASEDIR"
id="propertyBaseDir"/>
<!-- Define JCloudStoreService bean -->
<!--
<bean name="jcloudStore" class="org.dspace.storage.bitstore.JCloudBitStoreService" scope="singleton" lazy-init="true">
<property name="enabled" value="${assetstore.s3.generic.enabled:false}"/>
<property name="useRelativePath" value="${assetstore.s3.generic.useRelativePath:false}"/>
<property name="container" value="${assetstore.s3.generic.container}"/>
<property name="endpoint" value="${assetstore.s3.endpoint}"/>
<property name="providerOrApi" value="${assetstore.s3.generic.provider}"/>
<property name="identity" value="${assetstore.s3.generic.awsAccessKey}"/>
<property name="credentials" value="${assetstore.s3.generic.awsSecretKey}"/>
<property name="maxCounter" value="${assetstore.s3.maxCounter:100}"/>
<property name="overrides">
<props>
<prop key="#{propertyBaseDir}">./local/filesystemstorage</prop>
</props>
</property>
Subfolder to organize assets within the bucket, in case this bucket is shared
Optional, default is root level of bucket
<property name="subFolder" value="${assetstore.s3.generic.subfolder}"/>
</bean>
-->
<!-- <bean name="localStore2 ... -->
<!-- <bean name="s3Store2 ... -->
</beans>

6
pom.xml
View File

@@ -1707,6 +1707,12 @@
<groupId>com.google.api-client</groupId>
<artifactId>google-api-client</artifactId>
<version>1.23.0</version>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava-jdk5</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.google.http-client</groupId>