hazza/DSpace
1
0
Fork 0
mirror of https://github.com/DSpace/DSpace.git synced 2025-10-07 10:04:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

DS-3651: MultipartFileSender with inputstreams

Browse Source
This commit is contained in:
frederic
2017-09-28 15:39:18 +02:00
committed by Tom Desair
parent b578b03595
commit 4cedffc3be
5 changed files with 599 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
dspace-spring-rest/src/main/java/org/dspace/app/rest/BitstreamContentRestController.java
View File

@@ -14,12 +14,10 @@ import java.util.UUID;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.io.IOUtils;
import org.dspace.app.rest.model.BitstreamRest;
import org.dspace.app.rest.repository.BitstreamRestRepository;
import org.dspace.core.Utils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.hateoas.Link;
import org.springframework.hateoas.ResourceSupport;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
@@ -35,18 +33,21 @@ import org.springframework.web.bind.annotation.RestController;
@RequestMapping("/api/"+BitstreamRest.CATEGORY +"/"+ BitstreamRest.PLURAL_NAME + "/{uuid:[0-9a-fxA-FX]{8}-[0-9a-fxA-FX]{4}-[0-9a-fxA-FX]{4}-[0-9a-fxA-FX]{4}-[0-9a-fxA-FX]{12}}/content")
public class BitstreamContentRestController {
@Autowired
private BitstreamRestRepository bitstreamRestRepository;
private BitstreamRestRepository bitstreamRestRepository;
private int buffer = 20480;
@RequestMapping(method = RequestMethod.GET)
public void retrieve(@PathVariable UUID uuid, HttpServletResponse response,
HttpServletRequest request) throws IOException {
HttpServletRequest request) throws IOException {
BitstreamRest bit = bitstreamRestRepository.findOne(uuid);
if (bit == null) {
response.sendError(HttpServletResponse.SC_NOT_FOUND);
return;
}
response.setHeader("ETag", bit.getCheckSum().getValue());
response.setContentLengthLong(bit.getSizeBytes());
// response.setContentLengthLong(bit.getSizeBytes());
// Check for if-modified-since header
long modSince = request.getDateHeader("If-Modified-Since");
// we should keep last modification date on the bitstream
@@ -57,15 +58,25 @@ public class BitstreamContentRestController {
// response.setStatus(HttpServletResponse.SC_NOT_MODIFIED);
// return;
// }
// Pipe the bits
InputStream is = bitstreamRestRepository.retrieve(uuid);
// Set the response MIME type
response.setContentType(bit.getFormat().getMimetype());
Utils.bufferedCopy(is, response.getOutputStream());
is.close();
response.getOutputStream().flush();
//MultipartFileSender
try {
MultipartFileSender.fromBitstream(bit).with(request).with(response).with(is).serveResource();
} catch (Exception e) {
e.printStackTrace();
} finally {
IOUtils.closeQuietly(is);
}
// is.close();
}
}

355
dspace-spring-rest/src/main/java/org/dspace/app/rest/MultipartFileSender.java Normal file
View File

@@ -0,0 +1,355 @@
package org.dspace.app.rest;
import org.apache.commons.lang3.StringUtils;
import org.apache.tika.Tika;
import org.dspace.app.rest.model.BitstreamRest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.attribute.FileTime;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
/**
* Created by kevin on 10/02/15.
* See full code here : https://github.com/davinkevin/Podcast-Server/blob/d927d9b8cb9ea1268af74316cd20b7192ca92da7/src/main/java/lan/dk/podcastserver/utils/multipart/MultipartFileSender.java
*/
public class MultipartFileSender {
protected final Logger logger = LoggerFactory.getLogger(this.getClass());
private static final int DEFAULT_BUFFER_SIZE = 20480; // ..bytes = 20KB.
private static final long DEFAULT_EXPIRE_TIME = 604800000L; // ..ms = 1 week.
private static final String MULTIPART_BOUNDARY = "MULTIPART_BYTERANGES";
BitstreamRest bitstream;
InputStream inputStream;
HttpServletRequest request;
HttpServletResponse response;
public MultipartFileSender() {
}
public static MultipartFileSender fromBitstream(BitstreamRest bitstream) {
return new MultipartFileSender().setBitstream(bitstream);
}
//** internal setter **//
private MultipartFileSender setBitstream(BitstreamRest bitstream) {
this.bitstream = bitstream;
return this;
}
public MultipartFileSender with(HttpServletRequest httpRequest) {
request = httpRequest;
return this;
}
public MultipartFileSender with(HttpServletResponse httpResponse) {
response = httpResponse;
return this;
}
public MultipartFileSender with(InputStream inputStream) {
this.inputStream = inputStream;
return this;
}
public void serveResource() throws Exception {
if (response == null || request == null) {
return;
}
// if (!Files.exists(filepath)) {
// logger.error("File doesn't exist at URI : {}", filepath.toAbsolutePath().toString());
// response.sendError(HttpServletResponse.SC_NOT_FOUND);
// return;
// }
Long length = bitstream.getSizeBytes();
String fileName = bitstream.getName();
FileTime lastModifiedObj = FileTime.fromMillis(new Date().getTime());
if (StringUtils.isEmpty(fileName) || lastModifiedObj == null) {
response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
return;
}
long lastModified = LocalDateTime.ofInstant(lastModifiedObj.toInstant(), ZoneId.of(ZoneOffset.systemDefault().getId())).toEpochSecond(ZoneOffset.UTC);
Tika tika = new Tika();
String contentType = tika.detect(inputStream);
// Validate request headers for caching ---------------------------------------------------
// If-None-Match header should contain "*" or ETag. If so, then return 304.
String ifNoneMatch = request.getHeader("If-None-Match");
if (ifNoneMatch != null && HttpUtils.matches(ifNoneMatch, fileName)) {
response.setHeader("ETag", fileName); // Required in 304.
response.sendError(HttpServletResponse.SC_NOT_MODIFIED);
return;
}
// If-Modified-Since header should be greater than LastModified. If so, then return 304.
// This header is ignored if any If-None-Match header is specified.
long ifModifiedSince = request.getDateHeader("If-Modified-Since");
if (ifNoneMatch == null && ifModifiedSince != -1 && ifModifiedSince + 1000 > lastModified) {
response.setHeader("ETag", fileName); // Required in 304.
response.sendError(HttpServletResponse.SC_NOT_MODIFIED);
return;
}
// Validate request headers for resume ----------------------------------------------------
// If-Match header should contain "*" or ETag. If not, then return 412.
String ifMatch = request.getHeader("If-Match");
if (ifMatch != null && !HttpUtils.matches(ifMatch, fileName)) {
response.sendError(HttpServletResponse.SC_PRECONDITION_FAILED);
return;
}
// If-Unmodified-Since header should be greater than LastModified. If not, then return 412.
long ifUnmodifiedSince = request.getDateHeader("If-Unmodified-Since");
if (ifUnmodifiedSince != -1 && ifUnmodifiedSince + 1000 <= lastModified) {
response.sendError(HttpServletResponse.SC_PRECONDITION_FAILED);
return;
}
// Validate and process range -------------------------------------------------------------
// Prepare some variables. The full Range represents the complete file.
Range full = new Range(0, length - 1, length);
List<Range> ranges = new ArrayList<>();
// Validate and process Range and If-Range headers.
String range = request.getHeader("Range");
if (range != null) {
// Range header should match format "bytes=n-n,n-n,n-n...". If not, then return 416.
if (!range.matches("^bytes=\\d*-\\d*(,\\d*-\\d*)*$")) {
response.setHeader("Content-Range", "bytes */" + length); // Required in 416.
response.sendError(HttpServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE);
return;
}
String ifRange = request.getHeader("If-Range");
if (ifRange != null && !ifRange.equals(fileName)) {
try {
long ifRangeTime = request.getDateHeader("If-Range"); // Throws IAE if invalid.
if (ifRangeTime != -1) {
ranges.add(full);
}
} catch (IllegalArgumentException ignore) {
ranges.add(full);
}
}
// If any valid If-Range header, then process each part of byte range.
if (ranges.isEmpty()) {
for (String part : range.substring(6).split(",")) {
// Assuming a file with length of 100, the following examples returns bytes at:
// 50-80 (50 to 80), 40- (40 to length=100), -20 (length-20=80 to length=100).
long start = Range.sublong(part, 0, part.indexOf("-"));
long end = Range.sublong(part, part.indexOf("-") + 1, part.length());
if (start == -1) {
start = length - end;
end = length - 1;
} else if (end == -1 || end > length - 1) {
end = length - 1;
}
// Check if Range is syntactically valid. If not, then return 416.
if (start > end) {
response.setHeader("Content-Range", "bytes */" + length); // Required in 416.
response.sendError(HttpServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE);
return;
}
// Add range.
ranges.add(new Range(start, end, length));
}
}
}
// Prepare and initialize response --------------------------------------------------------
// Get content type by file name and set content disposition.
String disposition = "inline";
// If content type is unknown, then set the default value.
// For all content types, see: http://www.w3schools.com/media/media_mimeref.asp
// To add new content types, add new mime-mapping entry in web.xml.
if (contentType == null) {
contentType = "application/octet-stream";
} else if (!contentType.startsWith("image")) {
// Else, expect for images, determine content disposition. If content type is supported by
// the browser, then set to inline, else attachment which will pop a 'save as' dialogue.
String accept = request.getHeader("Accept");
disposition = accept != null && HttpUtils.accepts(accept, contentType) ? "inline" : "attachment";
}
logger.debug("Content-Type : {}", contentType);
// Initialize response.
response.reset();
response.setBufferSize(DEFAULT_BUFFER_SIZE);
response.setHeader("Content-Type", contentType);
response.setHeader("Content-Disposition", disposition + ";filename=\"" + fileName + "\"");
logger.debug("Content-Disposition : {}", disposition);
response.setHeader("Accept-Ranges", "bytes");
response.setHeader("ETag", fileName);
response.setDateHeader("Last-Modified", lastModified);
response.setDateHeader("Expires", System.currentTimeMillis() + DEFAULT_EXPIRE_TIME);
// Send requested file (part(s)) to client ------------------------------------------------
// Prepare streams.
try (InputStream input = inputStream;
OutputStream output = response.getOutputStream()) {
if (ranges.isEmpty() || ranges.get(0) == full) {
// Return full file.
logger.info("Return full file");
response.setContentType(contentType);
response.setHeader("Content-Range", "bytes " + full.start + "-" + full.end + "/" + full.total);
response.setHeader("Content-Length", String.valueOf(full.length));
Range.copy(input, output, length, full.start, full.length);
} else if (ranges.size() == 1) {
// Return single part of file.
Range r = ranges.get(0);
logger.info("Return 1 part of file : from ({}) to ({})", r.start, r.end);
response.setContentType(contentType);
response.setHeader("Content-Range", "bytes " + r.start + "-" + r.end + "/" + r.total);
response.setHeader("Content-Length", String.valueOf(r.length));
response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT); // 206.
// Copy single part range.
Range.copy(input, output, length, r.start, r.length);
} else {
// Return multiple parts of file.
response.setContentType("multipart/byteranges; boundary=" + MULTIPART_BOUNDARY);
response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT); // 206.
// Cast back to ServletOutputStream to get the easy println methods.
ServletOutputStream sos = (ServletOutputStream) output;
// Copy multi part range.
for (Range r : ranges) {
logger.info("Return multi part of file : from ({}) to ({})", r.start, r.end);
// Add multipart boundary and header fields for every range.
sos.println();
sos.println("--" + MULTIPART_BOUNDARY);
sos.println("Content-Type: " + contentType);
sos.println("Content-Range: bytes " + r.start + "-" + r.end + "/" + r.total);
// Copy single part range of multi part range.
Range.copy(input, output, length, r.start, r.length);
}
// End with multipart boundary.
sos.println();
sos.println("--" + MULTIPART_BOUNDARY + "--");
}
}
}
private static class Range {
long start;
long end;
long length;
long total;
/**
* Construct a byte range.
* @param start Start of the byte range.
* @param end End of the byte range.
* @param total Total length of the byte source.
*/
public Range(long start, long end, long total) {
this.start = start;
if (end < start + DEFAULT_BUFFER_SIZE) {
this.end = end;
} else {
this.end = Math.min(start + DEFAULT_BUFFER_SIZE, total - 1);
}
this.length = this.end - start + 1;
this.total = total;
}
public static long sublong(String value, int beginIndex, int endIndex) {
String substring = value.substring(beginIndex, endIndex);
return (substring.length() > 0) ? Long.parseLong(substring) : -1;
}
private static void copy(InputStream input, OutputStream output, long inputSize, long start, long length) throws IOException {
byte[] buffer = new byte[DEFAULT_BUFFER_SIZE];
int read;
if (inputSize == length) {
// Write full range.
while ((read = input.read(buffer)) > 0) {
output.write(buffer, 0, read);
output.flush();
}
} else {
input.skip(start);
long toRead = length;
while ((read = input.read(buffer)) > 0) {
if ((toRead -= read) > 0) {
output.write(buffer, 0, read);
output.flush();
} else {
output.write(buffer, 0, (int) toRead + read);
output.flush();
break;
}
}
}
}
}
private static class HttpUtils {
/**
* Returns true if the given accept header accepts the given value.
* @param acceptHeader The accept header.
* @param toAccept The value to be accepted.
* @return True if the given accept header accepts the given value.
*/
public static boolean accepts(String acceptHeader, String toAccept) {
String[] acceptValues = acceptHeader.split("\\s*(,|;)\\s*");
Arrays.sort(acceptValues);
return Arrays.binarySearch(acceptValues, toAccept) > -1
|| Arrays.binarySearch(acceptValues, toAccept.replaceAll("/.*$", "/*")) > -1
|| Arrays.binarySearch(acceptValues, "*/*") > -1;
}
/**
* Returns true if the given match header matches the given value.
* @param matchHeader The match header.
* @param toMatch The value to be matched.
* @return True if the given match header matches the given value.
*/
public static boolean matches(String matchHeader, String toMatch) {
String[] matchValues = matchHeader.split("\\s*,\\s*");
Arrays.sort(matchValues);
return Arrays.binarySearch(matchValues, toMatch) > -1
|| Arrays.binarySearch(matchValues, "*") > -1;
}
}
}

16
dspace-spring-rest/src/main/java/org/dspace/app/rest/repository/BitstreamRestRepository.java
View File

@@ -7,14 +7,6 @@
*/
package org.dspace.app.rest.repository;
import java.io.IOException;
import java.io.InputStream;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.UUID;
import org.dspace.app.rest.converter.BitstreamConverter;
import org.dspace.app.rest.model.BitstreamRest;
import org.dspace.app.rest.model.hateoas.BitstreamResource;
@@ -28,6 +20,14 @@ import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.io.InputStream;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.UUID;
/**
* This is the repository responsible to manage Bitstream Rest object
*

166
dspace-spring-rest/src/main/java/org/dspace/app/rest/utils/BitstreamResourceHttpRequestHandler.java Normal file
View File

@@ -0,0 +1,166 @@
package org.dspace.app.rest.utils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.tika.Tika;
import org.springframework.core.io.Resource;
import org.springframework.core.io.support.ResourceRegion;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpRange;
import org.springframework.http.MediaType;
import org.springframework.http.server.ServletServerHttpRequest;
import org.springframework.http.server.ServletServerHttpResponse;
import org.springframework.web.context.request.ServletWebRequest;
import org.springframework.web.servlet.resource.ResourceHttpRequestHandler;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.util.List;
public class BitstreamResourceHttpRequestHandler extends ResourceHttpRequestHandler {
private static final Log logger = LogFactory.getLog(ResourceHttpRequestHandler.class);
private Resource resource;
/**
* Processes a resource request.
* <p>Checks for the existence of the requested resource in the configured list of locations.
* If the resource does not exist, a {@code 404} response will be returned to the client.
* If the resource exists, the request will be checked for the presence of the
* {@code Last-Modified} header, and its value will be compared against the last-modified
* timestamp of the given resource, returning a {@code 304} status code if the
* {@code Last-Modified} value is greater. If the resource is newer than the
* {@code Last-Modified} value, or the header is not present, the content resource
* of the resource will be written to the response with caching headers
* set to expire one year in the future.
*/
@Override
public void handleRequest(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
// For very general mappings (e.g. "/") we need to check 404 first
Resource resource = getResource(request);
if (resource == null) {
logger.trace("No matching resource found - returning 404");
response.sendError(HttpServletResponse.SC_NOT_FOUND);
return;
}
if (HttpMethod.OPTIONS.matches(request.getMethod())) {
response.setHeader("Allow", getAllowHeader());
return;
}
// Supported methods and required session
checkRequest(request);
// Header phase
if (new ServletWebRequest(request, response).checkNotModified(resource.lastModified())) {
logger.trace("Resource not modified - returning 304");
return;
}
// Apply cache settings, if any
prepareResponse(response);
// Check the media type for the resource
MediaType mediaType = getMediaType(request, resource);
if (mediaType != null) {
if (logger.isTraceEnabled()) {
logger.trace("Determined media type '" + mediaType + "' for " + resource);
}
}
else {
if (logger.isTraceEnabled()) {
logger.trace("No media type found for " + resource + " - not sending a content-type header");
}
}
// Content phase
if (METHOD_HEAD.equals(request.getMethod())) {
setHeaders(response, resource, mediaType);
logger.trace("HEAD request - skipping content");
return;
}
ServletServerHttpResponse outputMessage = new ServletServerHttpResponse(response);
if (request.getHeader(HttpHeaders.RANGE) == null) {
setHeaders(response, resource, mediaType);
getResourceHttpMessageConverter().write(resource, mediaType, outputMessage);
}
else {
response.setHeader(HttpHeaders.ACCEPT_RANGES, "bytes");
ServletServerHttpRequest inputMessage = new ServletServerHttpRequest(request);
try {
HttpHeaders headers = inputMessage.getHeaders();
/** CUSTOM: Limit range **/
List<HttpRange> httpRanges = headers.getRange();
httpRanges = limitRangeHeader(headers, httpRanges);
response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
if (httpRanges.size() == 1) {
ResourceRegion resourceRegion = httpRanges.get(0).toResourceRegion(resource);
getResourceRegionHttpMessageConverter().write(resourceRegion, mediaType, outputMessage);
}
else {
getResourceRegionHttpMessageConverter().write(
HttpRange.toResourceRegions(httpRanges, resource), mediaType, outputMessage);
}
}
catch (IllegalArgumentException ex) {
response.setHeader("Content-Range", "bytes */" + resource.contentLength());
response.sendError(HttpServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE);
}
}
}
private List<HttpRange> limitRangeHeader(HttpHeaders headers, List<HttpRange> httpRanges) throws IOException {
List<String> ranges = headers.get(HttpHeaders.RANGE);
if (ranges.get(0).trim().endsWith("-")) {
long start = httpRanges.get(0).getRangeStart(0);
long end = Math.min(start + 20480, resource.contentLength() - 1);
headers.set(HttpHeaders.RANGE, "bytes=" + start + "-" + end);
return headers.getRange();
}
return httpRanges;
}
public BitstreamResourceHttpRequestHandler(Resource resource) {
super();
this.resource = resource;
try {
afterPropertiesSet();
} catch (Exception e) {
e.printStackTrace();
}
}
@Override
protected Resource getResource(HttpServletRequest request) throws IOException {
return this.resource;
}
@Override
protected MediaType getMediaType(HttpServletRequest request, Resource resource) {
Tika tika = new Tika();
try {
String mimetype = tika.detect(resource.getInputStream());
return MediaType.parseMediaType(mimetype);
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
}

45
dspace-spring-rest/src/main/java/org/dspace/app/rest/utils/DSpaceRestInputStreamResource.java Normal file
View File

@@ -0,0 +1,45 @@
package org.dspace.app.rest.utils;
import org.springframework.core.io.InputStreamResource;
import java.io.IOException;
import java.io.InputStream;
import java.util.Date;
public class DSpaceRestInputStreamResource extends InputStreamResource {
private InputStream inputStream;
private long contentLength;
private String filename;
public DSpaceRestInputStreamResource(InputStream inputStream, Long contentLength, String filename) {
super(inputStream);
this.inputStream = inputStream;
this.contentLength = contentLength;
this.filename = filename;
}
@Override
public long contentLength() throws IOException {
return this.contentLength;
}
@Override
public String getFilename() {
return this.filename;
}
@Override
public long lastModified() throws IOException {
return new Date().getTime();
}
@Override
public InputStream getInputStream() throws IOException, IllegalStateException {
return this.inputStream;
}
}