diff --git a/dspace-api/src/main/java/org/dspace/content/Bitstream.java b/dspace-api/src/main/java/org/dspace/content/Bitstream.java index b0b6c210e4..254c0e57e4 100644 --- a/dspace-api/src/main/java/org/dspace/content/Bitstream.java +++ b/dspace-api/src/main/java/org/dspace/content/Bitstream.java @@ -7,11 +7,6 @@ */ package org.dspace.content; -import java.io.InputStream; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; - import org.dspace.content.factory.ContentServiceFactory; import org.dspace.content.service.BitstreamService; import org.dspace.core.Constants; @@ -19,6 +14,10 @@ import org.dspace.core.Context; import org.hibernate.proxy.HibernateProxyHelper; import javax.persistence.*; +import java.io.InputStream; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; /** * Class representing bitstreams stored in the DSpace system. diff --git a/dspace-api/src/main/java/org/dspace/content/BitstreamServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/BitstreamServiceImpl.java index 2ddf806eff..7eb3b5021c 100644 --- a/dspace-api/src/main/java/org/dspace/content/BitstreamServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/BitstreamServiceImpl.java @@ -27,9 +27,7 @@ import org.springframework.beans.factory.annotation.Autowired; import java.io.IOException; import java.io.InputStream; import java.sql.SQLException; -import java.util.Iterator; -import java.util.List; -import java.util.UUID; +import java.util.*; /** * Service implementation for the Bitstream object. @@ -453,4 +451,8 @@ public class BitstreamServiceImpl extends DSpaceObjectServiceImpl imp public List getNotReferencedBitstreams(Context context) throws SQLException { return bitstreamDAO.getNotReferencedBitstreams(context); } + + public String getLastModified(Bitstream bitstream) { + return bitstreamStorageService.getLastModified(bitstream); + } } diff --git a/dspace-api/src/main/java/org/dspace/content/service/BitstreamService.java b/dspace-api/src/main/java/org/dspace/content/service/BitstreamService.java index d536f5670c..ac5f24278c 100644 --- a/dspace-api/src/main/java/org/dspace/content/service/BitstreamService.java +++ b/dspace-api/src/main/java/org/dspace/content/service/BitstreamService.java @@ -202,4 +202,6 @@ public interface BitstreamService extends DSpaceObjectService, DSpace int countBitstreamsWithoutPolicy(Context context) throws SQLException; List getNotReferencedBitstreams(Context context) throws SQLException; + + public String getLastModified(Bitstream bitstream); } diff --git a/dspace-api/src/main/java/org/dspace/storage/bitstore/BitstreamStorageServiceImpl.java b/dspace-api/src/main/java/org/dspace/storage/bitstore/BitstreamStorageServiceImpl.java index 79f44e47eb..172c9674a7 100644 --- a/dspace-api/src/main/java/org/dspace/storage/bitstore/BitstreamStorageServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/storage/bitstore/BitstreamStorageServiceImpl.java @@ -339,6 +339,17 @@ public class BitstreamStorageServiceImpl implements BitstreamStorageService, Ini } } + public String getLastModified(Bitstream bitstream) { + Map wantedMetadata = new HashMap(); + wantedMetadata.put("modified", null); + try { + wantedMetadata = stores.get(incoming).about(bitstream, wantedMetadata); + } catch (IOException e) { + log.error(e); + } + return wantedMetadata.get("modified").toString(); + } + /** * * @param context diff --git a/dspace-api/src/main/java/org/dspace/storage/bitstore/service/BitstreamStorageService.java b/dspace-api/src/main/java/org/dspace/storage/bitstore/service/BitstreamStorageService.java index 5258e1cc81..8cc7d075bc 100644 --- a/dspace-api/src/main/java/org/dspace/storage/bitstore/service/BitstreamStorageService.java +++ b/dspace-api/src/main/java/org/dspace/storage/bitstore/service/BitstreamStorageService.java @@ -190,4 +190,7 @@ public interface BitstreamStorageService { */ public void migrate(Context context, Integer assetstoreSource, Integer assetstoreDestination, boolean deleteOld, Integer batchCommitSize) throws IOException, SQLException, AuthorizeException; + + public String getLastModified(Bitstream bitstream); + } diff --git a/dspace-spring-rest/src/main/java/org/dspace/app/rest/BitstreamContentRestController.java b/dspace-spring-rest/src/main/java/org/dspace/app/rest/BitstreamContentRestController.java index d9406d42f1..e002634e87 100644 --- a/dspace-spring-rest/src/main/java/org/dspace/app/rest/BitstreamContentRestController.java +++ b/dspace-spring-rest/src/main/java/org/dspace/app/rest/BitstreamContentRestController.java @@ -15,8 +15,6 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.commons.io.IOUtils; -import org.apache.tika.Tika; -import org.apache.tika.mime.MimeTypes; import org.dspace.app.rest.model.BitstreamRest; import org.dspace.app.rest.repository.BitstreamRestRepository; import org.dspace.app.rest.utils.MultipartFileSender; @@ -38,10 +36,7 @@ public class BitstreamContentRestController { @Autowired private BitstreamRestRepository bitstreamRestRepository; - private int buffer = 20480; - - - @RequestMapping(method = RequestMethod.GET) + @RequestMapping(method = {RequestMethod.GET, RequestMethod.HEAD}) public void retrieve(@PathVariable UUID uuid, HttpServletResponse response, HttpServletRequest request) throws IOException { BitstreamRest bit = bitstreamRestRepository.findOne(uuid); @@ -53,24 +48,26 @@ public class BitstreamContentRestController { // Pipe the bits InputStream is = bitstreamRestRepository.retrieve(uuid); + long lastModified = bitstreamRestRepository.getLastModified(uuid); + String mimetype = bit.getFormat().getMimetype(); - //This should be improved somewhere else so we don't have to look for the correct mimetype here - if (mimetype.equals(MimeTypes.OCTET_STREAM)) { - Tika tika = new Tika(); - mimetype = tika.detect(is); - is.close(); - is = bitstreamRestRepository.retrieve(uuid); - } + + //TODO LOG DOWNLOAD if no range or if last chunk + + //MultipartFileSender try { - MultipartFileSender.fromBitstream(bit).with(request).with(response).with(is).with(mimetype).serveResource(); + MultipartFileSender.fromBitstream(bit).with(request).with(response).withInputStream(is).withMimetype(mimetype).withLastModified(lastModified).serveResource(); } catch (Exception e) { e.printStackTrace(); } finally { IOUtils.closeQuietly(is); } + + } + } \ No newline at end of file diff --git a/dspace-spring-rest/src/main/java/org/dspace/app/rest/converter/BitstreamConverter.java b/dspace-spring-rest/src/main/java/org/dspace/app/rest/converter/BitstreamConverter.java index 2ad6760c10..e8ee16847a 100644 --- a/dspace-spring-rest/src/main/java/org/dspace/app/rest/converter/BitstreamConverter.java +++ b/dspace-spring-rest/src/main/java/org/dspace/app/rest/converter/BitstreamConverter.java @@ -7,9 +7,6 @@ */ package org.dspace.app.rest.converter; -import java.sql.SQLException; -import java.util.List; - import org.dspace.app.rest.model.BitstreamFormatRest; import org.dspace.app.rest.model.BitstreamRest; import org.dspace.app.rest.model.CheckSumRest; @@ -17,6 +14,9 @@ import org.dspace.content.Bundle; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import java.sql.SQLException; +import java.util.List; + /** * This is the converter from/to the Bitstream in the DSpace API data model and the REST data model diff --git a/dspace-spring-rest/src/main/java/org/dspace/app/rest/repository/BitstreamRestRepository.java b/dspace-spring-rest/src/main/java/org/dspace/app/rest/repository/BitstreamRestRepository.java index 8d13e979ec..810444e33c 100644 --- a/dspace-spring-rest/src/main/java/org/dspace/app/rest/repository/BitstreamRestRepository.java +++ b/dspace-spring-rest/src/main/java/org/dspace/app/rest/repository/BitstreamRestRepository.java @@ -110,4 +110,15 @@ public class BitstreamRestRepository extends DSpaceRestRepository lastModified) { - response.setHeader("ETag", fileName); // Required in 304. + long ifModifiedSince = request.getDateHeader(IF_MODIFIED_SINCE); + if (isNull(ifNoneMatch) && ifModifiedSince != -1 && ifModifiedSince + 1000 > lastModified) { + log.error("If-Modified-Since header should be greater than LastModified. If so, then return 304."); + response.setHeader(ETAG, bitstream.getCheckSum().getValue()); // Required in 304. response.sendError(HttpServletResponse.SC_NOT_MODIFIED); return; } @@ -114,15 +156,17 @@ public class MultipartFileSender { // Validate request headers for resume ---------------------------------------------------- // If-Match header should contain "*" or ETag. If not, then return 412. - String ifMatch = request.getHeader("If-Match"); - if (ifMatch != null && !HttpUtils.matches(ifMatch, fileName)) { + String ifMatch = request.getHeader(IF_MATCH); + if (nonNull(ifMatch) && !matches(ifMatch, fileName)) { + log.error("If-Match header should contain \"*\" or ETag. If not, then return 412."); response.sendError(HttpServletResponse.SC_PRECONDITION_FAILED); return; } // If-Unmodified-Since header should be greater than LastModified. If not, then return 412. - long ifUnmodifiedSince = request.getDateHeader("If-Unmodified-Since"); + long ifUnmodifiedSince = request.getDateHeader(IF_UNMODIFIED_SINCE); if (ifUnmodifiedSince != -1 && ifUnmodifiedSince + 1000 <= lastModified) { + log.error("If-Unmodified-Since header should be greater than LastModified. If not, then return 412."); response.sendError(HttpServletResponse.SC_PRECONDITION_FAILED); return; } @@ -134,20 +178,21 @@ public class MultipartFileSender { List ranges = new ArrayList<>(); // Validate and process Range and If-Range headers. - String range = request.getHeader("Range"); - if (range != null) { + String range = request.getHeader(RANGE); + if (nonNull(range)) { // Range header should match format "bytes=n-n,n-n,n-n...". If not, then return 416. if (!range.matches("^bytes=\\d*-\\d*(,\\d*-\\d*)*$")) { - response.setHeader("Content-Range", "bytes */" + length); // Required in 416. + log.error("Range header should match format \"bytes=n-n,n-n,n-n...\". If not, then return 416."); + response.setHeader(CONTENT_RANGE, String.format(BYTES_DINVALID_BYTE_RANGE_FORMAT, length)); // Required in 416. response.sendError(HttpServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE); return; } - String ifRange = request.getHeader("If-Range"); - if (ifRange != null && !ifRange.equals(fileName)) { + String ifRange = request.getHeader(IF_RANGE); + if (nonNull(ifRange) && !ifRange.equals(fileName)) { try { - long ifRangeTime = request.getDateHeader("If-Range"); // Throws IAE if invalid. + long ifRangeTime = request.getDateHeader(IF_RANGE); // Throws IAE if invalid. if (ifRangeTime != -1) { ranges.add(full); } @@ -158,6 +203,7 @@ public class MultipartFileSender { // If any valid If-Range header, then process each part of byte range. if (ranges.isEmpty()) { + log.info("If any valid If-Range header, then process each part of byte range."); for (String part : range.substring(6).split(",")) { // Assuming a file with length of 100, the following examples returns bytes at: // 50-80 (50 to 80), 40- (40 to length=100), -20 (length-20=80 to length=100). @@ -173,7 +219,8 @@ public class MultipartFileSender { // Check if Range is syntactically valid. If not, then return 416. if (start > end) { - response.setHeader("Content-Range", "bytes */" + length); // Required in 416. + log.info("Check if Range is syntactically valid. If not, then return 416."); + response.setHeader(CONTENT_RANGE, String.format(BYTES_DINVALID_BYTE_RANGE_FORMAT, length)); // Required in 416. response.sendError(HttpServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE); return; } @@ -184,57 +231,62 @@ public class MultipartFileSender { } } - // Prepare and initialize response -------------------------------------------------------- - - // Get content type by file name and set content disposition. - String disposition = "inline"; - - // If content type is unknown, then set the default value. - // For all content types, see: http://www.w3schools.com/media/media_mimeref.asp - // To add new content types, add new mime-mapping entry in web.xml. - if (contentType == null) { - contentType = "application/octet-stream"; - } else if (!contentType.startsWith("image")) { - // Else, expect for images, determine content disposition. If content type is supported by - // the browser, then set to inline, else attachment which will pop a 'save as' dialogue. - String accept = request.getHeader("Accept"); - disposition = accept != null && HttpUtils.accepts(accept, contentType) ? "inline" : "attachment"; - } - logger.debug("Content-Type : {}", contentType); + log.debug("Content-Type : {}", contentType); // Initialize response. response.reset(); response.setBufferSize(DEFAULT_BUFFER_SIZE); - response.setHeader("Content-Type", contentType); - response.setHeader("Content-Disposition", disposition + ";filename=\"" + fileName + "\""); - logger.debug("Content-Disposition : {}", disposition); - response.setHeader("Accept-Ranges", "bytes"); - response.setHeader("ETag", fileName); - response.setDateHeader("Last-Modified", lastModified); - response.setDateHeader("Expires", System.currentTimeMillis() + DEFAULT_EXPIRE_TIME); + response.setHeader(CONTENT_TYPE, contentType); + response.setHeader(ACCEPT_RANGES, BYTES); + response.setHeader(ETAG, bitstream.getCheckSum().getValue()); + response.setDateHeader(LAST_MODIFIED, lastModified); + response.setDateHeader(EXPIRES, System.currentTimeMillis() + DEFAULT_EXPIRE_TIME); + //TODO what does this mean + response.setHeader(CACHE_CONTROL, "private, no-cache"); + + + if (isNullOrEmpty(disposition)) { + if (contentType == null) { + contentType = APPLICATION_OCTET_STREAM; + } else if (!contentType.startsWith(IMAGE)) { + String accept = request.getHeader(ACCEPT); + disposition = accept != null && accepts(accept, contentType) ? CONTENT_DISPOSITION_INLINE : CONTENT_DISPOSITION_ATTACHMENT; + } + + response.setHeader(CONTENT_DISPOSITION, String.format(CONTENT_DISPOSITION_FORMAT, disposition, fileName)); + log.debug("Content-Disposition : {}", disposition); + } + + + // Content phase + if (METHOD_HEAD.equals(request.getMethod())) { + log.debug("HEAD request - skipping content"); + return; + } // Send requested file (part(s)) to client ------------------------------------------------ // Prepare streams. try (InputStream input = inputStream; OutputStream output = response.getOutputStream()) { + if (ranges.isEmpty() || ranges.get(0) == full) { // Return full file. - logger.info("Return full file"); + log.info("Return full file"); response.setContentType(contentType); - response.setHeader("Content-Range", "bytes " + full.start + "-" + full.end + "/" + full.total); - response.setHeader("Content-Length", String.valueOf(full.length)); + response.setHeader(CONTENT_RANGE, String.format(BYTES_RANGE_FORMAT, full.start, full.end, full.total)); + response.setHeader(CONTENT_LENGTH, String.valueOf(full.length)); Range.copy(input, output, length, full.start, full.length); } else if (ranges.size() == 1) { // Return single part of file. Range r = ranges.get(0); - logger.info("Return 1 part of file : from ({}) to ({})", r.start, r.end); + log.info("Return 1 part of file : from ({}) to ({})", r.start, r.end); response.setContentType(contentType); - response.setHeader("Content-Range", "bytes " + r.start + "-" + r.end + "/" + r.total); - response.setHeader("Content-Length", String.valueOf(r.length)); + response.setHeader(CONTENT_RANGE, String.format(BYTES_RANGE_FORMAT, r.start, r.end, r.total)); + response.setHeader(CONTENT_LENGTH, String.valueOf(r.length)); response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT); // 206. // Copy single part range. @@ -243,20 +295,20 @@ public class MultipartFileSender { } else { // Return multiple parts of file. - response.setContentType("multipart/byteranges; boundary=" + MULTIPART_BOUNDARY); + response.setContentType(CONTENT_TYPE_MULTITYPE_WITH_BOUNDARY); response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT); // 206. // Cast back to ServletOutputStream to get the easy println methods. ServletOutputStream sos = (ServletOutputStream) output; // Copy multi part range. - for (Range r : ranges) { - logger.info("Return multi part of file : from ({}) to ({})", r.start, r.end); + for (Range r : Range.relativize(ranges)) { + log.info("Return multi part of file : from ({}) to ({})", r.start, r.end); // Add multipart boundary and header fields for every range. sos.println(); sos.println("--" + MULTIPART_BOUNDARY); - sos.println("Content-Type: " + contentType); - sos.println("Content-Range: bytes " + r.start + "-" + r.end + "/" + r.total); + sos.println(CONTENT_TYPE + ": " + contentType); + sos.println(CONTENT_RANGE + ": " + String.format(BYTES_RANGE_FORMAT, r.start, r.end, r.total)); // Copy single part range of multi part range. Range.copy(input, output, length, r.start, r.length); @@ -268,8 +320,14 @@ public class MultipartFileSender { } } + } + private static boolean isNullOrEmpty(String disposition) { + return !(disposition == null || disposition.length() == 0); + } + + private static class Range { long start; long end; @@ -278,21 +336,36 @@ public class MultipartFileSender { /** * Construct a byte range. + * * @param start Start of the byte range. - * @param end End of the byte range. + * @param end End of the byte range. * @param total Total length of the byte source. */ public Range(long start, long end, long total) { this.start = start; - if (end < start + DEFAULT_BUFFER_SIZE) { +// if (end <= start + DEFAULT_BUFFER_SIZE) { this.end = end; - } else { - this.end = Math.min(start + DEFAULT_BUFFER_SIZE, total - 1); - } +// } else { +// this.end = Math.min(start + DEFAULT_BUFFER_SIZE, total - 1); +// } this.length = this.end - start + 1; this.total = total; } + private static List relativize(List ranges) { + + List builder = new ArrayList<>(ranges.size()); + + Range prevRange = null; + for (Range r : ranges) { + Range newRange = isNull(prevRange) ? r : new Range(r.start - prevRange.end - 1, r.end - prevRange.end - 1, r.total); + builder.add(newRange); + prevRange = r; + } + + return builder; + } + public static long sublong(String value, int beginIndex, int endIndex) { String substring = value.substring(beginIndex, endIndex); return (substring.length() > 0) ? Long.parseLong(substring) : -1; @@ -325,34 +398,20 @@ public class MultipartFileSender { } } } - private static class HttpUtils { - /** - * Returns true if the given accept header accepts the given value. - * @param acceptHeader The accept header. - * @param toAccept The value to be accepted. - * @return True if the given accept header accepts the given value. - */ - public static boolean accepts(String acceptHeader, String toAccept) { - String[] acceptValues = acceptHeader.split("\\s*(,|;)\\s*"); - Arrays.sort(acceptValues); + private static boolean accepts(String acceptHeader, String toAccept) { + String[] acceptValues = acceptHeader.split("\\s*(,|;)\\s*"); + Arrays.sort(acceptValues); - return Arrays.binarySearch(acceptValues, toAccept) > -1 - || Arrays.binarySearch(acceptValues, toAccept.replaceAll("/.*$", "/*")) > -1 - || Arrays.binarySearch(acceptValues, "*/*") > -1; - } - - /** - * Returns true if the given match header matches the given value. - * @param matchHeader The match header. - * @param toMatch The value to be matched. - * @return True if the given match header matches the given value. - */ - public static boolean matches(String matchHeader, String toMatch) { - String[] matchValues = matchHeader.split("\\s*,\\s*"); - Arrays.sort(matchValues); - return Arrays.binarySearch(matchValues, toMatch) > -1 - || Arrays.binarySearch(matchValues, "*") > -1; - } + return Arrays.binarySearch(acceptValues, toAccept) > -1 + || Arrays.binarySearch(acceptValues, toAccept.replaceAll("/.*$", "/*")) > -1 + || Arrays.binarySearch(acceptValues, "*/*") > -1; } + + private static boolean matches(String matchHeader, String toMatch) { + String[] matchValues = matchHeader.split("\\s*,\\s*"); + Arrays.sort(matchValues); + return Arrays.binarySearch(matchValues, toMatch) > -1 || Arrays.binarySearch(matchValues, "*") > -1; + } + } \ No newline at end of file diff --git a/dspace/config/dspace.cfg b/dspace/config/dspace.cfg index 83e950ce92..82fa2021e4 100644 --- a/dspace/config/dspace.cfg +++ b/dspace/config/dspace.cfg @@ -1962,6 +1962,8 @@ mail.helpdesk = ${mail.admin} # Should all Request Copy emails go to the helpdesk instead of the item submitter? request.item.helpdesk.override = false +#bitstream-download.buffer.size = 1000000 + #------------------------------------------------------------------# diff --git a/dspace/config/registries/bitstream-formats.xml b/dspace/config/registries/bitstream-formats.xml index ba532add0e..f098aaa0c8 100644 --- a/dspace/config/registries/bitstream-formats.xml +++ b/dspace/config/registries/bitstream-formats.xml @@ -736,4 +736,22 @@ epub + + video/mp4 + mp4 + mpeg4 + 1 + false + mp4 + + + + audio/mpeg + mp3 + MPEG audio + 1 + false + mp3 + +